diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..623998fd --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,22 @@ +## Description + + + +## Related Issues + + + +## Checklist + +- [ ] I have read and followed the [CONTRIBUTING.md](https://github.com/Security-Onion-Solutions/securityonion/blob/3/main/CONTRIBUTING.md) file. +- [ ] I have read and agree to the terms of the [Contributor License Agreement](https://securityonionsolutions.com/cla) + +## Questions or Comments + + \ No newline at end of file diff --git a/.github/workflows/contrib.yml b/.github/workflows/contrib.yml deleted file mode 100644 index 2cbdb278..00000000 --- a/.github/workflows/contrib.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: contrib -on: - issue_comment: - types: [created] - pull_request_target: - types: [opened,closed,synchronize] - -jobs: - CLAssistant: - runs-on: ubuntu-latest - steps: - - name: "Contributor Check" - if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' - uses: cla-assistant/github-action@v2.3.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} - with: - path-to-signatures: 'signatures_v1.json' - path-to-document: 'https://securityonionsolutions.com/cla' - allowlist: dependabot[bot],jertel,dougburks,TOoSmOotH,defensivedepth,m0duspwnens - remote-organization-name: Security-Onion-Solutions - remote-repository-name: licensing - diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8c9ac19d..30c7319b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -4,6 +4,8 @@ on: branches: - main - dev + - '*/dev' + - '*/main' jobs: deploy: runs-on: ubuntu-latest @@ -18,15 +20,30 @@ jobs: with: python-version: '3.x' - - run: pip install mkdocs mkdocs-material mkdocs-glightbox + - run: pip install mkdocs mkdocs-material mkdocs-glightbox mkdocs-to-pdf - - run: mkdocs build --strict + - run: | + echo "VERSION=$(curl -f https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/refs/heads/3/dev/VERSION)" >> $GITHUB_ENV + shell: bash + + - run: echo "Found version $VERSION" + + - run: mkdocs build + + - run: npx -y @redocly/cli build-docs specs/openapi.yaml -o /home/runner/work/docs/docs/site/connect-api/so-api-reference.html + + - name: Generate GitHub App Token + id: generate_token + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} - name: Deploy to GitHub Pages uses: JamesIves/github-pages-deploy-action@v4 with: folder: site # mkdocs default output dir branch: gh-pages # your Pages source branch - token: ${{ secrets.GITHUB_TOKEN }} # automatic, no need to set manually - clean: false # IMPORTANT: false prevents wiping other folders (like dev/) - target-folder: ${{ github.ref_name == 'main' && '.' || github.ref_name }} + token: ${{ steps.generate_token.outputs.token }} + clean: false # IMPORTANT: false prevents wiping other folders + target-folder: ${{ github.ref_name == 'main' && '.' || format('en/{0}', github.ref_name) }} diff --git a/.gitignore b/.gitignore index 28b21a96..8aebba50 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ dist json .venv +site +_build diff --git a/README.md b/README.md index ae0f64be..d7b73ac9 100644 --- a/README.md +++ b/README.md @@ -1 +1,5 @@ -# securityonion-3-docs +# Security Onion Docs + +The docs in this repo are hosted via Github Pages at: + +https://security-onion-solutions.github.io/docs/ diff --git a/docs/_static/mkdocsoad.css b/docs/_static/mkdocsoad.css new file mode 100644 index 00000000..39425de5 --- /dev/null +++ b/docs/_static/mkdocsoad.css @@ -0,0 +1,191 @@ +/** + * CSS for OpenAPI HTML generated with PyMdown Extensions option. + * + * This CSS file works when using the OAD plugin with pymdownx. + * See here how to use it: + * https://www.neoteroi.dev/mkdocs-plugins/web/oad/ + * + * https://github.com/Neoteroi/mkdocs-plugins +**/ + +:root { + --http-get-color: green; + --http-delete-color: #dc0101; + --http-head-color: slateblue; + --http-options-color: steelblue; + --http-patch-color: darkorange; + --http-post-color: darkblue; + --http-put-color: darkmagenta; + --http-trace-color: darkcyan; + --http-route-param-color: rgb(51, 128, 210); + --oad-operation-separator-border-color: gray; + --oad-block-border-color: #00bfa5; + --oad-small-note-color: #666; + --oad-indent-border-color: #c5c5c5; +} + +@media screen { + /* Slate theme, i.e. dark mode */ + [data-md-color-scheme="slate"] { + --http-get-color: #2ea82e; + --http-post-color: #0093c0; + --http-put-color: #c333c3; + --oad-small-note-color: #afafaf; + } +} + +.api-tag { + font-weight: bold; +} + +span[class^="http-"] { + font-weight: bold; + color: #fff; + padding: 4px 1rem; + border-radius: 2px; + margin-right: .5rem; +} + +.http-get { + background-color: var(--http-get-color); +} + +.http-delete { + background-color: var(--http-delete-color); +} + +.http-post { + background-color: var(--http-post-color); +} + +.http-patch { + background-color: var(--http-patch-color); +} + +.http-trace { + background-color: var(--http-trace-color); +} + +.http-put { + background-color: var(--http-put-color); +} + +.http-head { + background-color: var(--http-head-color); +} + +.http-options { + background-color: var(--http-options-color); +} + +.route-param { + color: var(--http-route-param-color); +} + +.operation-separator + h3[id^="get"] .route-param { + color: var(--http-get-color); +} + +.operation-separator + h3[id^="delete"] .route-param { + color: var(--http-delete-color); +} + + +.operation-separator + h3[id^="post"] .route-param { + color: var(--http-post-color); +} + +.operation-separator + h3[id^="patch"] .route-param { + color: var(--http-patch-color); +} + +.operation-separator + h3[id^="trace"] .route-param { + color: var(--http-trace-color); +} + +.operation-separator + h3[id^="put"] .route-param { + color: var(--http-put-color); +} + +.operation-separator + h3[id^="head"] .route-param { + color: var(--http-head-color); +} + +.operation-separator + h3[id^="options"] .route-param { + color: var(--http-options-color); +} + +.api-version { + font-size: 1.2rem; +} + +.operation-separator { + margin: 0 !important; + border-bottom: 2px dotted var(--oad-operation-separator-border-color) !important; + padding-top: .5rem; +} + +.operation-separator + h3 { + margin-top: 1rem; +} + +.string-type { + color: var(--md-code-hl-string-color); +} + +.integer-type, .number-type { + color: var(--md-code-hl-number-color); +} + +.boolean-type { + color: var(--md-code-hl-keyword-color); +} + +.format { + color: var(--md-code-hl-name-color); +} + +.null-type { + color: var(--md-code-hl-keyword-color); +} + +a.ref-link { + color: var(--md-code-hl-special-color); +} + +.request-block + div { + padding-left: 1rem; + border-left: 2px dashed var(--oad-block-border-color); +} + +.small-note { + font-size: 14px; + color: var(--oad-small-note-color); +} + +.request-body-title { + margin-bottom: 4px; +} + +.request-body-title + .tabbed-set, +.response-title + .tabbed-set, +.message-separator + .tabbed-set, +.common-response, +.response-section { + margin-top: 2px; + padding-left: 1rem; + border-left: 2px dotted var(--oad-indent-border-color); +} + +.info-data { + font-size: .6rem; +} + +.message-separator { + visibility: hidden; +} + +.sub-section-title { + font-style: italic; + font-size: 14px; +} \ No newline at end of file diff --git a/docs/_static/theme_overrides.css b/docs/_static/theme_overrides.css new file mode 100644 index 00000000..5c6c0335 --- /dev/null +++ b/docs/_static/theme_overrides.css @@ -0,0 +1,28 @@ +/* PDF defaults to A4 - change to letter portrait and set margins */ +@page { + size: letter portrait !important; + margin: 0.75in 0.7in 0.75in 0.7in !important; +} + +/* PDF needs table styles to fit within margins */ +@media print { + table { + table-layout: fixed !important; + width: 100% !important; + border-collapse: collapse !important; + font-size: 5pt !important; + line-height: 1.1 !important; + } + + th, td { + padding: 1px 2px !important; + font-size: 5pt !important; + } +} + +/* Both HTML and PDF - add white background to diagram images for better visibility on dark themes */ +img[src*="images/diagrams/"] { + background-color: white; + padding: 10px; + border-radius: 4px; +} diff --git a/docs/_static/theme_overrides.js b/docs/_static/theme_overrides.js new file mode 100644 index 00000000..f4ef20fe --- /dev/null +++ b/docs/_static/theme_overrides.js @@ -0,0 +1 @@ +// Theme overrides JavaScript diff --git a/docs/about.md b/docs/about.md deleted file mode 100644 index e8195edb..00000000 --- a/docs/about.md +++ /dev/null @@ -1,56 +0,0 @@ -# About - -## Security Onion - -Security Onion is a free and open platform built by defenders for defenders. It includes [network visibility](network-visibility.md), [host visibility](host-visibility.md), [intrusion detection honeypots](idh.md), [log management](elasticsearch.md), and [case management](cases.md). Security Onion has been downloaded over 2 million times and is being used by security teams around the world to monitor and defend their enterprises. Our easy-to-use Setup wizard allows you to build a distributed Grid for your enterprise in minutes! - -## Security Onion Solutions, LLC - -Doug Burks started Security Onion as a free and open project in 2008 and then founded Security Onion Solutions, LLC in 2014. - -!!! IMPORTANT - - Security Onion Solutions, LLC is the only official provider of hardware appliances, training, and professional services for Security Onion. - -For more information about these products and services, please see our company site at . - -## Documentation - -!!! WARNING - - Documentation is always a work in progress and some documentation may be missing or incorrect. Please let us know if you notice any issues. - -### License - -This documentation is licensed under CC BY 4.0. You can read more about this license at . - -### Formats - -This documentation is published online at . If you are viewing an offline version of this documentation but have Internet access, you might want to switch to the online version at to see the latest version. - -This documentation is also available in PDF format at . - -Many folks have asked for a printed version of our documentation. Whether you work on airgapped networks or simply want a portable reference that doesn't require an Internet connection or batteries, this is what you've been asking for. Thanks to Richard Bejtlich for writing the inspiring foreword! Proceeds go to the Rural Technology Fund! You can purchase your copy at . - -### Authors - -Security Onion Solutions is the primary author and maintainer of this documentation. Some content has been contributed by members of our community. Thanks to all the folks who have contributed to this documentation over the years! - -### Contributing - -We welcome your contributions to our documentation! We will review any suggestions and apply them if appropriate. - -If you are accessing the online version of the documentation and notice that a particular page has incorrect information, you can submit corrections by clicking the `Edit on GitHub` button in the upper-right corner of each page. Once you have made your corrections, you will need to submit your pull request (PR) to the `dev` branch. - -To submit a new page, you can submit a pull request (PR) to the `dev` branch of the `securityonion-docs` repo at . - -Pages are written in Markdown format and you can find several Markdown guides on the Internet including . - -### Naming Convention - -New documentation pages should use the following naming convention: - -- all lowercase -- `.md` file extension -- ideally, the name of the page should be one simple word (for example: `suricata.md`) -- if necessary, the name of the page can be hyphenated (for example: network-visibility.md) diff --git a/docs/accounts.md b/docs/accounts.md index 26768477..a9da77f0 100644 --- a/docs/accounts.md +++ b/docs/accounts.md @@ -1,4 +1,4 @@ -# Accounts +# Accounts Overview In Security Onion, there are two main types of accounts: @@ -6,13 +6,3 @@ In Security Onion, there are two main types of accounts: - application accounts used when authenticating to [SOC](security-onion-console.md) OS accounts are controlled by standard Linux account utilities. SOC accounts are maintained via the [Administration](administration.md) interface. If for some reason you can't log into SOC, you can use [so-user](so-user.md) from the command line. - -## Table of Contents - -- [Passwords](passwords.md) -- [MFA](mfa.md) -- [Adding Accounts](adding-accounts.md) -- [Listing Accounts](listing-accounts.md) -- [Disabling Accounts](disabling-accounts.md) -- [RBAC](rbac.md) -- [Kratos](kratos.md) \ No newline at end of file diff --git a/docs/active-query-management.md b/docs/active-query-management.md index 179159b8..fdcab9ca 100644 --- a/docs/active-query-management.md +++ b/docs/active-query-management.md @@ -4,7 +4,7 @@ This is an enterprise-level feature of Security Onion. Contact Security Onion Solutions, LLC via our website at for more information about purchasing a Security Onion Pro license to enable this feature. -Starting in version 2.4.130, Security Onion Pro customers can now view and cancel long-running Elasticsearch queries. +Security Onion Pro customers can now view and cancel long-running Elasticsearch queries. This screen is located under the Administration menu on the left side of the Security Onion Console. This menu option will only be visible for users having the `superuser` role. @@ -41,4 +41,4 @@ a short time. !!! WARNING Canceling internal, system queries can disrupt the Elasticsearch internal processes. Avoid canceling queries that are not - confirmed to be user-created. \ No newline at end of file + confirmed to be user-created. diff --git a/docs/additional-network-visibility.md b/docs/additional-network-visibility.md index 68f0bb26..274f1514 100644 --- a/docs/additional-network-visibility.md +++ b/docs/additional-network-visibility.md @@ -1,14 +1,5 @@ -# Additional Network Visibility +# Additional Network Visibility Overview -In the [network](network-visibility.md) section, we looked at network visibility provided by Security Onion itself. The ideal situation would be to have Security Onion network sensors covering each and every one of your network segments. If you're able to achieve that ideal situation, then you may not need any additional network visibility. However, there may be times when you simply can't cover certain network segments with Security Onion network sensors and that's when these additional options can be beneficial. Keep in mind, though, that the data that they provide is nowhere near as comprehensive as a full Security Onion network sensor. +In the [Network Visibility](network-visibility.md) section, we looked at network visibility provided by Security Onion itself. The ideal situation would be to have Security Onion network sensors covering each and every one of your network segments. If you're able to achieve that ideal situation, then you may not need any additional network visibility. However, there may be times when you simply can't cover certain network segments with Security Onion network sensors and that's when these additional options can be beneficial. Keep in mind, though, that the data that they provide is nowhere near as comprehensive as a full Security Onion network sensor. One option for additional network visibility would be [NetFlow](netflow.md) logs from firewalls, switches, or routers showing what traffic was observed by the network device. Another option would be firewall logs showing what traffic was allowed through the firewall and what traffic was denied. Firewall logs may be in different formats such as [CEF](cef.md), [iptables](iptables.md), or a combination of the two (as in [UniFi](unifi.md) firewalls). We also have support for [pfSense](pfsense.md) and [OPNsense](opnsense.md) firewalls and you can find other firewall integrations in the [Third Party Integrations](third-party-integrations.md) section. - -## Table of Contents - -- [NetFlow](netflow.md) -- [CEF](cef.md) -- [iptables](iptables.md) -- [UniFi](unifi.md) -- [pfSense](pfsense.md) -- [OPNsense](opnsense.md) \ No newline at end of file diff --git a/docs/administration.md b/docs/administration.md index ecb420ad..85cb5984 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -1,10 +1,10 @@ # Administration -[SOC](security-onion-console.md) includes an Administration section which allows you to administer Users, Grid Members, Configuration, and the License Key. +[Security Onion Console](security-onion-console.md) includes an Administration section which allows you to administer Users, Grid Members, Configuration, and the License Key. ## Users -The Users page shows all user accounts that have been created for the Grid. +The Users page shows all user accounts that have been created for the grid. ![Image](images/81_users.png) @@ -23,17 +23,17 @@ Hovering over the icon in the Status column will show you these details as well. ## Grid Members -The Grid Members page shows nodes that have attempted to join the Grid and whether or not they have been accepted into the Grid by an administrator. +The Grid Members page shows nodes that have attempted to join the grid and whether or not they have been accepted into the grid by an administrator. ![Image](images/84_gridmembers.png) Unaccepted members are displayed on the left side and broken into three sections: Pending Members, Denied Members, and Rejected Members. When you accept a member, it will then move to the right side under Accepted Members. -For accepted members, you can click the REVIEW button to show additional information about the Grid member. If you want to remove the member, you can then click the DELETE button and review the confirmation. +For accepted members, you can click the REVIEW button to show additional information about the grid member. If you want to remove the member, you can then click the DELETE button and review the confirmation. ## Configuration -The Configuration page allows you to configure various components of your Grid. +The Configuration page allows you to configure various components of your grid. ![Image](images/87_config.png) @@ -45,13 +45,13 @@ If unsure of which component a particular setting may belong to, use the Filter - expand all settings - collapse all settings - show settings that have been modified from the default value -- show settings that have a unique value specified for one or more nodes in the Grid +- show settings that have a unique value specified for one or more nodes in the grid !!! NOTE Keys that include `_x_` indicate a placeholder value used to represent a period (`.`). -Some settings can be applied across the entire Grid or to specific nodes. Applying a setting to a specific node will override the Grid setting. +Some settings can be applied across the entire Grid or to specific nodes. Applying a setting to a specific node will override the grid setting. ### Advanced Settings @@ -80,4 +80,4 @@ Some settings can be duplicated to more easily create new settings. If a setting ![Image](images/91_licensekey.png) -The License Key screen allows you to add a license key for [Security Onion Pro](security-onion-pro.md). Once you've added a license key, the screen will show details about your license key. \ No newline at end of file +The License Key screen allows you to add a license key for [Security Onion Pro](security-onion-pro.md). Once you've added a license key, the screen will show details about your license key. diff --git a/docs/airgap.md b/docs/airgap.md index 4fc288d9..d2fb70aa 100644 --- a/docs/airgap.md +++ b/docs/airgap.md @@ -2,13 +2,13 @@ Security Onion is committed to allowing users to run a full install on networks that do not have Internet access. Our ISO image includes everything you need to run without Internet access. Make sure that you choose the airgap option during Setup. -If your network has Internet access but has overly restrictive proxies, firewalls, or other network devices that might prevent Security Onion from connecting to the sites shown in the [firewall](firewall.md) section, then you may want to consider the airgap option as everything will install from the ISO image itself. +If your network has Internet access but has overly restrictive proxies, firewalls, or other network devices that might prevent Security Onion from connecting to the sites shown in the [Firewall](firewall.md) section, then you may want to consider the airgap option as everything will install from the ISO image itself. ![Image](images/06_setup_airgap.png) Airgap mode works as follows: -- During the install, all of the necessary RPM packages are copied from the ISO image to a new repo located in `/nsm/repo/`. All devices in the Grid will now use this repo for updates to packages. +- During the install, all of the necessary RPM packages are copied from the ISO image to a new repo located in `/nsm/repo/`. All devices in the grid will now use this repo for updates to packages. - [NIDS](nids.md) rules for [Suricata](suricata.md) are copied to `/nsm/rules/suricata`. @@ -26,4 +26,4 @@ Our ISO image includes the latest version of various rulesets and will automatic - [YARA](yara.md): Most recent rules from our repo -- [Sigma](sigma.md): Most recent rule packages from the SigmaHQ repo \ No newline at end of file +- [Sigma](sigma.md): Most recent rule packages from the SigmaHQ repo diff --git a/docs/alert-data-fields.md b/docs/alert-data-fields.md index d3cd926b..5db6aca8 100644 --- a/docs/alert-data-fields.md +++ b/docs/alert-data-fields.md @@ -8,9 +8,9 @@ You can find these online at: -- -- -- +- +- +- You can find parsed [NIDS](nids.md) alerts in [Alerts](alerts.md), [Dashboards](dashboards.md), [Hunt](hunt.md), and [Kibana](kibana.md) via their predefined queries and dashboards or by manually searching for: @@ -30,4 +30,4 @@ Those alerts should have the following fields: - `rule.rev` - `rule.severity` - `rule.uuid` -- `rule.version` \ No newline at end of file +- `rule.version` diff --git a/docs/alerts.md b/docs/alerts.md index e47d2ba3..6d009544 100644 --- a/docs/alerts.md +++ b/docs/alerts.md @@ -1,6 +1,6 @@ # Alerts -[SOC](security-onion-console.md) includes an Alerts interface which gives you an overview of the alerts that Security Onion is generating. You can then quickly drill down into details, pivot to [Hunt](hunt.md) or the [PCAP](pcap.md) interface, and escalate alerts to [Cases](cases.md). +[Security Onion Console](security-onion-console.md) includes an Alerts interface which gives you an overview of the alerts that Security Onion is generating. You can then quickly drill down into details, pivot to [Hunt](hunt.md) or the [PCAP](pcap.md) interface, and escalate alerts to [Cases](cases.md). ![Image](images/50_alerts.png) @@ -82,7 +82,7 @@ For more information about Playbooks, please see the [Detections](detections.md) !!! WARNING - Guided Analysis is a new experimental feature. Some of these playbooks were generated by AI and it's possible that they may not be 100% accurate. Please let us know if you see any issues. + Some playbooks were generated by AI and it's possible that they may not be 100% accurate. Please let us know if you see any issues. ![Image](images/51_alerts_play.png) @@ -162,4 +162,4 @@ The `Actions` sub-menu has several different options. Please note that some of t - Clicking the `Process Ancestors` option will show all parent processes for the selected process. This option will only appear if you click on a log that contains the `process.Ext.ancestry` field. -If you'd like to add your own custom actions, see the [SOC Customization](security-onion-console-customization.md) section. \ No newline at end of file +If you'd like to add your own custom actions, see the [SOC Customization](security-onion-console-customization.md) section. diff --git a/docs/appendix.md b/docs/appendix.md index 38ea7fd2..67f97fad 100644 --- a/docs/appendix.md +++ b/docs/appendix.md @@ -1,6 +1,6 @@ # Appendix -This appendix provides an overview of the process of migrating from the old Security Onion 2.3 to the new Security Onion 2.4. +This appendix provides an overview of the process of upgrading from the old Security Onion 2.4 to the new Security Onion 3. !!! TIP @@ -8,143 +8,50 @@ This appendix provides an overview of the process of migrating from the old Secu !!! WARNING - Security Onion 2.4 is a MAJOR change, so please note the following: - - - Security Onion 2.4 has higher hardware requirements, so you should check that your hardware meets those requirements. - - The /nsm partition must be on a separate disk. - - InfluxDB data is not migrated. - - If you have a distributed deployment, please note that 2.3 search nodes defaulted to cross cluster search whereas 2.4 defaults to full Elastic clustering. This means that you may need to rename or delete some Elasticsearch indices. - - We do not provide any guarantees that the upgrade process will work! If the upgrade fails, be prepared to perform a fresh installation of Security Onion 2.4. - -For the reasons listed above, we recommend that most users procure new hardware and perform a fresh installation of Security Onion 2.4. - -!!! TIP - - If you're planning to purchase new hardware, please consider official Security Onion appliances from Security Onion Solutions (). Our custom appliances have already been designed for certain roles and traffic levels and have Security Onion 2 pre-installed. + Security Onion 3 only supports Oracle Linux 9. If you are running Security Onion 2.4 on some other unsupported distro, then you will need to perform a fresh installation of Security Onion 3. !!! WARNING We recommend trying this process in a test environment before attempting in your production environment. - -!!! WARNING - Please ensure that you have local access to the machine being upgraded via console, DRAC, IPMI, etc. Failure to do so could result in an unsuccessful upgrade, requiring a clean installation of Security Onion 2.4. -If you have reviewed all of the warnings above and still want to attempt migration, you should be able to do the following. - -!!! NOTE - - If you have a distributed deployment, you will need to perform the steps on the manager first and then on each of the remaining nodes. - -First, make sure that your 2.3 installation is fully updated via [soup](soup.md): - -``` -sudo soup -``` +## Check Backup -Next, make sure there is a backup in /nsm/backup: +If you have reviewed all of the warnings above, next make sure there is a backup in /nsm/backup: ``` sudo ls -alh /nsm/backup ``` -Disable services and reboot: - -``` -sudo systemctl disable salt-minion -sudo reboot -``` - -Make sure docker containers are stopped: - -``` -sudo su -c "systemctl stop docker docker.socket" -sudo docker ps -``` - -If there are any remaining docker processes, stop them (replacing `$CONT_ID` with the actual ID): - -``` -sudo docker stop $CONT_ID -``` - -This can also be done in one command: - -``` -sudo docker ps | awk '!/CONTAINER/ { system("sudo docker stop " $1 ) }' -``` - -Unmount /nsm: - -``` -sudo umount /nsm -sudo vgchange -an /dev/mapper/nsm -sudo vgexport /dev/mapper/nsm -``` - -Boot the Security Onion 2.4 ISO image and go through the initial OS installation as shown in the [installation](installation.md) section. +Once you have checked your backup and are ready to upgrade, you should be able to do the following. There are separate sections for Internet deployments and Airgap deployments. -!!! WARNING - - During the OS installation, do NOT select the old NSM disk! It will be re-imported and used AFTER the OS install with LVM. +## Internet Deployments -After installation and reboot, log in as the user you created during installation. Setup will automatically start, but you'll need to cancel setup and change partitioning (replacing `/home/user/` with your desired temporary location): +If your deployment has Internet access, first make sure that it is fully updated to version 2.4.211 via [soup](soup.md): ``` -sudo cp -av /nsm/* /home/user/ -sudo umount /nsm -sudo lvremove /dev/system/nsm - -sudo lvresize -L +XG /dev/system/root -sudo xfs_growfs /dev/system/root - -sudo vgimport /dev/mapper/nsm -sudo vgchange -ay /dev/mapper/nsm -``` - -Add entry into /etc/fstab and then mount: - -``` -sudo mount -a -sudo systemctl daemon-reload -``` - -Remove /nsm/repo and /nsm/docker-registry from the old 2.3 /nsm. - -Copy the /nsm contents of /home/user/ (or wherever they were copied to) back to /nsm -(repo, docker-registry, and Elastic Fleet) - -Run through setup as described in the [Configuration](configuration.md) section. - -After setup, get the secrets pillar from /nsm/backup (replacing `2023_08_30` with the date of your most recent backup): - -``` -tar -xvf /nsm/backup/so-config-backup-2023_08_30.tar opt/so/saltstack/local/pillar/secrets.sls +sudo soup ``` -Replace the mysql secret in secrets.sls with the backed-up value: +Once your deployment has been upgraded to 2.4.211, then run the following command: ``` -docker exec -it so-mysql mysql -u root -p -# when prompted, enter the password from the 2.3 secrets.sls +sudo soupto3 ``` -At the mysql prompt, run the following query: +This command will check some settings on your system and then download a new version of soup. You can then run the new version of soup to perform the upgrade: ``` -SELECT User, Host from mysql.user; +sudo soup ``` -If you get the error `mysql error 1130: 172.17.1.1' is not allowed to connect to this mysql server`, then run the following: +## Airgap Deployments -``` -UPDATE mysql.user SET Host = '172.17.1.1' WHERE User = 'root' AND Host = 'localhost'; -``` +If your deployment does not have Internet access and is in airgap mode, first make sure that it is fully updated to version 2.4.211 using [soup](soup.md#airgap) and the 2.4.211 ISO image. -Exit the mysql shell and restart the so-mysql container. +Once that is done, then you will need to manually do a few things to prepare for the upgrade: -Run a full checkin: +- make sure that the ``pcapengine`` setting is set to ``SURICATA`` +- delete any old stenographer data -``` -sudo so-checkin -``` \ No newline at end of file +Finally, you can upgrade to version 3.0 using [soup](soup.md#airgap) and the Security Onion 3 ISO image. diff --git a/docs/architecture.md b/docs/architecture.md index 42d31950..0d16f140 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -100,7 +100,7 @@ Receiver nodes were designed with 2 purposes in mind: - reduce the load on the manager - offer pipeline redundancy -Each receiver node runs [Logstash](logstash.md) and [Redis](redis.md) and allows for events to continue to be processed by search nodes in the event the manager node is offline. When a receiver node joins the Grid, [Elastic Agent](elastic-agent.md) on all nodes adds this new address as a load balanced [Logstash](logstash.md) output. The search nodes add this new node as another [Logstash](logstash.md) input. Receiver nodes are "active-active" and you can add as many as you want (within reason) and events will be balanced among them. +Each receiver node runs [Logstash](logstash.md) and [Redis](redis.md) and allows for events to continue to be processed by search nodes in the event the manager node is offline. When a receiver node joins the grid, [Elastic Agent](elastic-agent.md) on all nodes adds this new address as a load balanced [Logstash](logstash.md) output. The search nodes add this new node as another [Logstash](logstash.md) input. Receiver nodes are "active-active" and you can add as many as you want (within reason) and events will be balanced among them. ![Image](images/diagrams/receiver.png) @@ -112,7 +112,7 @@ Search nodes connect to both the manager and receiver nodes and pull events from If you have a manager or managersearch that is under heavy load due to handling a high volume of events, then system resources can be freed by directing the Elastic Agent to only output events to the receiver node(s) in the environment. Once all configurable and advanced settings are enabled, this feature can be set in SOC Configuration UI under `elasticfleet > enable_manager_output`. Setting this to `False` will prevent the Elastic Agent from sending events to the manager, managersearch, or standalone nodes. -Receiver nodes need to be close to the search nodes because when you add a new receiver node to the Grid, the search nodes add the [Redis](redis.md) service as an input in their configs automatically. If you were to place a receiver node at a remote site, then ALL of your search nodes would be trying to access that [Redis](redis.md) queue remotely. You do not save any bandwidth by placing a receiver node at a remote site. +Receiver nodes need to be close to the search nodes because when you add a new receiver node to the grid, the search nodes add the [Redis](redis.md) service as an input in their configs automatically. If you were to place a receiver node at a remote site, then ALL of your search nodes would be trying to access that [Redis](redis.md) queue remotely. You do not save any bandwidth by placing a receiver node at a remote site. There are a couple of things to be aware of regarding receiver nodes and Elastic Agents. The first is Fleet which handles things like updating the agents and scheduling searches. The other is the Elastic Agent log output, which in this case is [Logstash](logstash.md) running on the manager or receiver node. Due to limitations in Elastic licensing we can only have a single output policy. That means that when you add a receiver or a fleet node it gets added to a list that is distributed to the agents. The agents go down that list and stop after a successful connection. The only way to direct agents to specific receivers is to use firewall rules to block agents to certain receivers. Again keep in mind that there is no bandwidth savings here because the search nodes still need to empty the [Redis](redis.md) queue on the receiver nodes. @@ -152,4 +152,4 @@ There are two instances of Elastic Agent that run on a Heavy Node: Instance 1 - Not connected to Fleet (runs standalone), runs in a container, picks up /nsm/ logs and other local logs (SOC) and sends them to the local Heavy Node ES cluster. -Instance 2 - Connected to Grid Fleet Server, runs directly on the Heavy Node. Not currently picking up any logs, but has the Osquery integration installed. \ No newline at end of file +Instance 2 - Connected to Grid Fleet Server, runs directly on the Heavy Node. Not currently picking up any logs, but has the Osquery integration installed. diff --git a/docs/attack-navigator.md b/docs/attack-navigator.md index de2d7164..d9c3c307 100644 --- a/docs/attack-navigator.md +++ b/docs/attack-navigator.md @@ -1,6 +1,6 @@ # ATT&CK Navigator -[SOC](security-onion-console.md) includes a link on the sidebar that takes you to ATT&CK Navigator. +[Security Onion Console](security-onion-console.md) includes a link on the sidebar that takes you to ATT&CK Navigator. From : @@ -16,11 +16,11 @@ To access Navigator, log into [SOC](security-onion-console.md) and then click th ## Configuration -Navigator reads its configuration from `/opt/so/conf/navigator/`. However, please keep in mind that if you make any changes here they may be overwritten since the config is managed with [salt](salt.md). +Navigator reads its configuration from `/opt/so/conf/navigator/`. However, please keep in mind that if you make any changes here they may be overwritten since the config is managed with [Salt](salt.md). ## More Information !!! NOTE For more information about ATT&CK Navigator, please see: - \ No newline at end of file + diff --git a/docs/backup.md b/docs/backup.md index e272ae4b..6ea7d0fa 100644 --- a/docs/backup.md +++ b/docs/backup.md @@ -1,13 +1,13 @@ # Backup -Security Onion performs a daily backup of some critical files so that you can recover your Grid from a catastophic failure of the manager. Daily backups create a tar file located in the `/nsm/backup/` directory located on the manager. You may want to replicate this backup directory to a location outside of your manager in case the manager ever needs to be rebuilt. +Security Onion performs a daily backup of some critical files so that you can recover your grid from a catastophic failure of the manager. Daily backups create a tar file located in the `/nsm/backup/` directory located on the manager. You may want to replicate this backup directory to a location outside of your manager in case the manager ever needs to be rebuilt. Here is what gets backed up automatically by default: - `/etc/pki/` - All of the certs including the CA. -- `/etc/salt/` - Configuration for the [salt](salt.md) manager and minions. -- `/nsm/kratos/` - Configuration for [kratos](kratos.md). -- `/nsm/hydra/` - Configuration for Hydra (used for [Connect](connect-api.md)). +- `/etc/salt/` - Configuration for the [Salt](salt.md) manager and minions. +- `/nsm/kratos/` - Configuration for [Kratos](kratos.md). +- `/nsm/hydra/` - Configuration for Hydra (used for [Security Onion API access](connect-api.md)). - `/opt/so/saltstack/local/` - Customizations done via [Administration](administration.md) --> Configuration. If you need to restore one or more files from backup, locate the tar backup file from the desired date and use the standard `tar` command to expand the file. For example, to expand the backup file from March 17, 2025: @@ -32,4 +32,14 @@ You can configure backups by going to [Administration](administration.md) --> Co Another option is to use [Elasticsearch](elasticsearch.md)'s built-in support for snapshots: -This option requires that you configure [Elasticsearch](elasticsearch.md) with a `path.repo` setting where it can store the snapshots. Once [Elasticsearch](elasticsearch.md) has the `path.repo` setting, you should be able to log into [Kibana](kibana.md) and configure snapshots as shown in the link above. Those snapshots will then be accessible in `/nsm/elasticsearch/repo/`. \ No newline at end of file +This option requires that you configure [Elasticsearch](elasticsearch.md) with a `path.repo` setting where it can store the snapshots. Once [Elasticsearch](elasticsearch.md) has the `path.repo` setting, you should be able to log into [Kibana](kibana.md) and configure snapshots as shown in the link above. Those snapshots will then be accessible in `/nsm/elasticsearch/repo/`. + +## PostgreSQL + +[PostgreSQL](postgresql.md) is automatically backed up daily at 00:05. The backup uses `pg_dumpall` to capture all databases and roles, compressed with gzip. Backup files are stored at `/nsm/backup/so-postgres-backup-YYYY_MM_DD.sql.gz` with 7-day retention and mode 0600. + +To restore from a backup: + +```bash +zcat /nsm/backup/so-postgres-backup-2026_04_21.sql.gz | docker exec -i so-postgres psql -U postgres +``` diff --git a/docs/best-practices.md b/docs/best-practices.md index 65ef6bca..2b50b8c8 100644 --- a/docs/best-practices.md +++ b/docs/best-practices.md @@ -4,9 +4,9 @@ Security Onion provides lots of options and flexibility, but for best results we ## Installation -- Download and verify our ISO image as shown in the [download](download.md) section. +- Download and verify our ISO image as shown in the [Download](download.md) section. -- For production deployments, prefer dedicated hardware to VMs when possible (see the [hardware](hardware.md) section). +- For production deployments, prefer dedicated hardware to VMs when possible (see the [Hardware](hardware.md) section). - If VMs must be used, ensure that resources are properly dedicated to VMs to avoid resource contention. @@ -16,7 +16,7 @@ Security Onion provides lots of options and flexibility, but for best results we - When possible, we recommend using a dedicated TAP rather than SPAN ports. -- Make sure that any network firewalls have the proper firewall rules in place to allow ongoing operation and updates (see the [firewall](firewall.md) section). +- Make sure that any network firewalls have the proper firewall rules in place to allow ongoing operation and updates (see the [Firewall](firewall.md) section). ## Configuration @@ -30,9 +30,9 @@ Security Onion provides lots of options and flexibility, but for best results we - Security Onion is a free and open platform based on standard Linux distros, but we recommend treating it as an appliance and avoid installing third party software as this may conflict with our components and cause issues when updating. -- Avoid installing automation tools such as Puppet and Chef as these may conflict with our existing [salt](salt.md) automation. +- Avoid installing automation tools such as Puppet and Chef as these may conflict with our existing [Salt](salt.md) automation. -- Avoid installing monitoring tools such as Zabbix as this may conflict with our existing [influxdb](influxdb.md) monitoring. +- Avoid installing monitoring tools such as Zabbix as this may conflict with our existing [Influxdb](influxdb.md) monitoring. - Avoid installing third-party endpoint security agents as they may break functionality or introduce unacceptable performance overhead. @@ -46,4 +46,4 @@ Security Onion provides lots of options and flexibility, but for best results we - Keep your deployment updated as we frequently fix bugs and add new features. -- If possible, test updates on a test deployment before deploying to production. \ No newline at end of file +- If possible, test updates on a test deployment before deploying to production. diff --git a/docs/bpf.md b/docs/bpf.md index ece084fc..d624dc53 100644 --- a/docs/bpf.md +++ b/docs/bpf.md @@ -62,8 +62,11 @@ not host 192.168.1.6 or not host 192.168.1.27 ### Troubleshooting BPF using tcpdump If you need to troubleshoot BPF, you can use `tcpdump` as shown in the following articles: + + + ## More Information @@ -71,5 +74,7 @@ If you need to troubleshoot BPF, you can use `tcpdump` as shown in the following !!! NOTE For more information about BPF, please see: + - \ No newline at end of file + + diff --git a/docs/cases.md b/docs/cases.md index 2601bfc2..06226905 100644 --- a/docs/cases.md +++ b/docs/cases.md @@ -1,6 +1,6 @@ # Cases -[SOC](security-onion-console.md) includes our Cases interface for case management. It allows you to escalate logs from [Alerts](alerts.md), [Dashboards](dashboards.md), and [Hunt](hunt.md), and then assign analysts, add comments and attachments, and track observables. +[Security Onion Console](security-onion-console.md) includes our Cases interface for case management. It allows you to escalate logs from [Alerts](alerts.md), [Dashboards](dashboards.md), and [Hunt](hunt.md), and then assign analysts, add comments and attachments, and track observables. On a new deployment, Cases will be empty until you create a new case. Once you have one or more cases, you can use the main Cases page to get an overview of all cases. @@ -130,7 +130,7 @@ The following is a summary of the built-in analyzers and their supported data ty !!! NOTE - The `malwarehashregistry` analyzer is no longer working as of 2.4.100. This is due to a stale third-party library that is incompatible with the latest Python version. See [#13571](https://github.com/Security-Onion-Solutions/securityonion/issues/13571) + The `malwarehashregistry` analyzer is no longer working. This is due to a stale third-party library that is incompatible with the latest Python version. See [#13571](https://github.com/Security-Onion-Solutions/securityonion/issues/13571) ### Running Analyzers @@ -173,10 +173,10 @@ At the top of the page, click the `Options` menu and then enable the `Show advan ### Developing Analyzers -If you'd like to develop a custom analyzer, take a look at the developer's guide at . +If you'd like to develop a custom analyzer, take a look at the developer's guide at . ## Templates SOC can use case templates to auto-populate default values of new cases. A template is itself a case, with its category set to `template`. To utilize that template case, the new case should specify the template case ID in the `template` field of the case object. -SOC automatically populates new case template fields with the value stored in the `rule.case_template` field of the alert being escalated. This allows for specific templates to be assigned to certain detection rules. For example, if alerts triggered from a certain rule are known to require a consistent set of resolution steps then the description of a case template can be prepopulated with that checklist (in markdown format). Then, the backing rule that triggered the alert can have its `case_template` field set to that case template ID. \ No newline at end of file +SOC automatically populates new case template fields with the value stored in the `rule.case_template` field of the alert being escalated. This allows for specific templates to be assigned to certain detection rules. For example, if alerts triggered from a certain rule are known to require a consistent set of resolution steps then the description of a case template can be prepopulated with that checklist (in markdown format). Then, the backing rule that triggered the alert can have its `case_template` field set to that case template ID. diff --git a/docs/cef.md b/docs/cef.md index ff25c321..90ce77ab 100644 --- a/docs/cef.md +++ b/docs/cef.md @@ -10,7 +10,7 @@ First, add the Elastic integration for `CEF`. For more information about the `CEF` integration, please see . -1. Go to [Elastic Fleet](elastic-fleet.md), click the `Agent policies` tab, and then click the desired policy (for example `so-Grid-nodes_general`). +1. Go to [Elastic Fleet](elastic-fleet.md), click the `Agent policies` tab, and then click the desired policy (for example `so-grid-nodes_general`). 2. Click the `Add integration` button. 3. Search for `cef` and then click on the `Common Event Format (CEF)` integration. 4. The Elastic Integration page will show an overview of the CEF Integration. Review all information on the page and then click the `Add Common Event Format (CEF)` button. @@ -29,7 +29,7 @@ Next, allow the traffic from the CEF host through the firewall to the CEF integr 3. On the left side, go to `firewall`, select `hostgroups`, and click the `customhostgroup0` group. On the right side, enter the IP address of the CEF host and click the checkmark to save. 4. On the left side, go to `firewall`, select `portgroups`, select the `customportgroup0` group, and then click `udp`. On the right side, enter your desired listener port (9003 by default) and click the checkmark to save. 5. On the left side, go to `firewall`, select `role`, and then select the node type that will receive the CEF logs. Then drill into `chain` --> `INPUT` --> `hostgroups` --> `customhostgroup0` --> `portgroups`. On the right side, enter `customportgroup0` and click the checkmark to save. -6. If you would like to apply the rules immediately, click the `SYNCHRONIZE Grid` button under the `Options` menu at the top of the page. +6. If you would like to apply the rules immediately, click the `SYNCHRONIZE GRID` button under the `Options` menu at the top of the page. ## CEF dashboard diff --git a/docs/cheat-sheet.md b/docs/cheat-sheet.md index a4425788..21608594 100644 --- a/docs/cheat-sheet.md +++ b/docs/cheat-sheet.md @@ -1,7 +1,7 @@ # Cheat Sheet -If you are viewing the online version of this documentation, you can [click here for our Security Onion Cheat Sheet](https://github.com/security-onion-solutions/securityonion-docs/raw/2.4/images/cheat-sheet/security-onion-cheat-sheet.pdf). +If you are viewing the online version of this documentation, you can [click here for our Security Onion Cheat Sheet](https://github.com/Security-Onion-Solutions/docs/raw/gh-pages/en/3/main/images/cheat-sheet/Security-Onion-Cheat-Sheet.pdf). This was based on a cheat sheet originally created by [Chris Sanders](https://chrissanders.org/) which can be found here: - \ No newline at end of file + diff --git a/docs/cloud-amazon.md b/docs/cloud-amazon.md index 8ff8f966..977f26af 100644 --- a/docs/cloud-amazon.md +++ b/docs/cloud-amazon.md @@ -1,11 +1,11 @@ # Amazon Cloud Image If you would like to deploy Security Onion in Amazon Web Services (AWS), we have an Amazon Machine Image (AMI) that is already built for you: - + !!! WARNING - Existing 2.4 RC1 or newer Security Onion AMI installations should use the [Soup](soup.md) command to upgrade to newer versions of Security Onion. Attempting to switch to a newer AMI from the AWS Marketplace could cause loss of data and require full Grid re-installation. Upgrading from Security Onion 2.3 or beta versions of 2.4 is unsupported. + Existing Security Onion cloud image installations should use the [soup](soup.md) command to upgrade. If your grid is still running 2.4.x, use ``soup`` to upgrade to 2.4.210, and then use ``soupto3`` to proceed to Security Onion 3, after which continue using ``soup`` again. Attempting to switch to a newer Security Onion image from the cloud marketplace could cause loss of data and require full Grid re-installation; use the ``soup`` procedure to upgrade instead. !!! NOTE @@ -17,7 +17,7 @@ If you would like to deploy Security Onion in Amazon Web Services (AWS), we have ## Requirements -Before proceeding, determine the Grid architecture desired. Choose from a single-node Grid versus a distributed, multi-node Grid. Additionally, determine if the lower latency of ephemeral instance storage is needed (typically when there is high-volume of traffic being monitored, which is most production scenarios), or if network-based storage, EBS, can be used for increased redundancy. +Before proceeding, determine the grid architecture desired. Choose from a single-node Grid versus a distributed, multi-node Grid. Additionally, determine if the lower latency of ephemeral instance storage is needed (typically when there is high-volume of traffic being monitored, which is most production scenarios), or if network-based storage, EBS, can be used for increased redundancy. ## Single Node Grid @@ -28,7 +28,7 @@ Listed below are the minimum suggested single-node instance quantities, sizes, a Standalone: - Quantity: 1 -- Type: t3a.xlarge +- Type: t3a.2xlarge - Storage: 256GB EBS (Optimized) gp3 Evaluation @@ -53,13 +53,13 @@ VPN Node Manager - Quantity: 1 -- Type: m5a.xlarge +- Type: m5a.2xlarge - Storage: 300GB EBS (Optimized) gp3 Search Nodes - Quantity: 2 or more -- Type: m5ad.xlarge +- Type: m5ad.2xlarge - Storage: 200GB EBS (Optimized) gp3 - Storage: 150GB Instance Storage (SSD/NVMe) @@ -174,9 +174,9 @@ Location: Remote Location: Remote Location: AWS Location: A 192.168.33.13 192.168.33.10 10.55.1.10 10.55.1.20 ``` -In order to add the Remote Network Sensor Node to the Grid, you would have to add `10.55.1.10` to the `sensor` firewall hostgroup. +In order to add the Remote Network Sensor Node to the grid, you would have to add `10.55.1.10` to the `sensor` firewall hostgroup. -This change can be done in the SOC Configuration screen. Then, either wait up to 15 minutes for the scheduled configuration sync to run, or force a synchronization immediately via the SOC Configuration Options. Once the firewall hostgroup configuration has been synchronized your Manager will be ready for remote minions to start connecting. +This change can be done in the SOC Configuration screen. [Auto State Apply](salt.md#auto-state-apply) should apply the new firewall hostgroup configuration within a few minutes, after which your Manager will be ready for remote minions to start connecting. ## AWS Traffic Mirroring @@ -230,4 +230,4 @@ To verify [Zeek](zeek.md) is properly decapsulating and parsing the VXLAN traffi ``` ls -la /nsm/zeek/logs/current/ -``` \ No newline at end of file +``` diff --git a/docs/cloud-azure.md b/docs/cloud-azure.md index 31d82ba3..5d1fdfe3 100644 --- a/docs/cloud-azure.md +++ b/docs/cloud-azure.md @@ -5,7 +5,7 @@ Azure users can deploy an official Security Onion virtual machine image found on !!! WARNING - Existing 2.4 RC1 or newer Security Onion Azure Image installations should use the [Soup](soup.md) command to upgrade to newer versions of Security Onion. Attempting to switch to a newer image from the Azure Marketplace could cause loss of data and require full Grid re-installation. Upgrading from Security Onion 2.3 or beta versions of 2.4 is unsupported. + Existing Security Onion cloud image installations should use the [soup](soup.md) command to upgrade. If your grid is still running 2.4.x, use ``soup`` to upgrade to 2.4.210, and then use ``soupto3`` to proceed to Security Onion 3, after which continue using ``soup`` again. Attempting to switch to a newer Security Onion image from the cloud marketplace could cause loss of data and require full Grid re-installation; use the ``soup`` procedure to upgrade instead. !!! NOTE @@ -21,7 +21,7 @@ Azure users can deploy an official Security Onion virtual machine image found on ## Requirements -Before proceeding, determine the Grid architecture desired. Choose from a single-node Grid versus a distributed, multi-node Grid. +Before proceeding, determine the grid architecture desired. Choose from a single-node Grid versus a distributed, multi-node Grid. Security Onion recommends using either Premium SSD disks, or the more expensive Ultra SSD disks, with suitable IOPS and throughput matched to your expected network monitoring requirements. @@ -34,7 +34,7 @@ Listed below are the minimum suggested single-node instance quantities, sizes, a ### Standalone - Quantity: 1 -- Type: Standard_D4as_v4 +- Type: Standard_D8as_v4 - Storage: 256GB Premium SSD ### Evaluation @@ -59,13 +59,13 @@ Listed below are the minimum suggested distributed Grid instance quantities, siz ### Manager - Quantity: 1 -- Type: Standard_D4as_v4 +- Type: Standard_D8as_v4 - Storage: 256GB Premium SSD ### Search Nodes - Quantity: 2 or more -- Type: Standard_D4as_v4 +- Type: Standard_D8as_v4 - Storage: 256GB Premium SSD ### Sensor monitoring the VPN ingress @@ -121,14 +121,14 @@ To configure a Security Onion instance (repeat for each node in a distributed Gr - Choose or create a new Resource group. - Enter a suitable name for this virtual machine, such as `so-vm-manager`. - Choose the desired Region and Availability options. (Use `East US 2` for Ultra SSD support, if needed.) -- Choose the `Security Onion 2 VM Image`. If this option is not listed on the Image dropdown, select `See all images` and search for `onion`. +- Choose the `Security Onion VM Image`. If this option is not listed on the Image dropdown, select `See all images` and search for `onion`. - Choose the appropriate Size based on the desired hardware requirements. For assistance on determining resource requirements please review the Requirements section above. - Change the Username to `onion`. Note that this is not mandatory -- if you accidentally leave it to the default `azureuser`, that's ok, you'll simply use the `azureuser` username any place where the documentation states to use the `onion` username. - Select an existing SSH public key if one already exists, otherwise select the option to `Generate new key pair`. - Choose `Other` for Licensing type. - Select `Next: Disks` - Ensure `Premium SSD` is selected. -- For single-node grids, distributed sensor nodes, or distributed search nodes: If you would like to separate the `/nsm` partition into its own disk, create and attach a data disk for this purpose, with a minimum size of 100GB, or more depending on predicted storage needs. Note that the size of the `/nsm` partition determines the rate that old packet and event data is pruned. Separating the /nsm partition can provide more flexibility with scaling up the Grid node sizes, but requires a little more setup, which is described later. +- For single-node grids, distributed sensor nodes, or distributed search nodes: If you would like to separate the `/nsm` partition into its own disk, create and attach a data disk for this purpose, with a minimum size of 100GB, or more depending on predicted storage needs. Note that the size of the `/nsm` partition determines the rate that old packet and event data is pruned. Separating the /nsm partition can provide more flexibility with scaling up the grid node sizes, but requires a little more setup, which is described later. - Select `Next: Networking` - Choose the virtual network for this virtual machine. - Choose a public IP if you intend to access this virtual machine directly (not recommended for production grids). @@ -179,4 +179,4 @@ To verify [Zeek](zeek.md) is properly decapsulating and parsing the traffic you ``` ls -la /nsm/zeek/logs/current/ -``` \ No newline at end of file +``` diff --git a/docs/cloud-google.md b/docs/cloud-google.md index c50600d2..7e3a8187 100644 --- a/docs/cloud-google.md +++ b/docs/cloud-google.md @@ -1,11 +1,11 @@ # Google Cloud Image -If you would like to deploy Security Onion in Google Cloud Platform (GCP), choose the Security Onion 2 image listed on the Google Marketplace: - +If you would like to deploy Security Onion in Google Cloud Platform (GCP), choose the Security Onion image listed on the Google Marketplace: + !!! WARNING - Existing 2.4 RC1 or newer Security Onion Google Image installations should use the [Soup](soup.md) command to upgrade to newer versions of Security Onion. Attempting to switch to a newer image from the Google Marketplace could cause loss of data and require full Grid re-installation. Upgrading from Security Onion 2.3 or beta versions of 2.4 is unsupported. + Existing Security Onion AMI installations should use the [soup](soup.md) command to upgrade. If your grid is still running 2.4.x, use ``soup`` to upgrade to 2.4.210, and then use ``soupto3`` to proceed to Security Onion 3, after which continue using ``soup`` again. Attempting to switch to a newer Security Onion image from the cloud marketplace could cause loss of data and require full Grid re-installation; use the ``soup`` procedure to upgrade instead. !!! NOTE @@ -17,7 +17,7 @@ If you would like to deploy Security Onion in Google Cloud Platform (GCP), choos ## Requirements -Before proceeding, determine the Grid architecture desired. Choose from a single-node Grid versus a distributed, multi-node Grid. Additionally, determine if the lower latency of local instance storage is needed (typically when there is high-volume of traffic being monitored, which is most production scenarios), or if persistent disks can be used for increased redundancy. +Before proceeding, determine the grid architecture desired. Choose from a single-node Grid versus a distributed, multi-node Grid. Additionally, determine if the lower latency of local instance storage is needed (typically when there is high-volume of traffic being monitored, which is most production scenarios), or if persistent disks can be used for increased redundancy. ## Single Node Grid @@ -28,7 +28,7 @@ Listed below are the minimum suggested single-node instance quantities, sizes, a ### Standalone - Quantity: 1 -- Type: n2-standard-4 +- Type: n2-standard-8 - Storage: 256GB Balanced Persistent Disk ### Evaluation @@ -54,13 +54,13 @@ Listed below are the minimum suggested distributed Grid instance quantities, siz ### Manager - Quantity: 1 -- Type: n2-standard-4 +- Type: n2-standard-8 - Storage: 300GB Balanced Persistent Disk ### Search Nodes - Quantity: 2 or more -- Type: n2-standard-4 +- Type: n2-standard-8 - Storage: 256GB Balanced Persistent Disk - Storage: 375GB Local Disk (NVMe) [optional] @@ -113,7 +113,7 @@ Traffic mirroring allows you to copy the traffic to/from an instance (or multipl Create a Packet Mirroring policy. This can be found in the Google Cloud Console under the VPC network section. When selecting the VPC network, choose the option that denotes the mirrored source and collector destination are in the same VPC network and select the Mirrored VPC network created earlier. -Under Select mirrored source, check the box next to the "Select with network tag" label. Then enter a tag named `so-mirror`. Once completed with the Grid setup, you can later tag all your VMs, whose traffic you want monitored, with the same `so-mirror` tag. +Under Select mirrored source, check the box next to the "Select with network tag" label. Then enter a tag named `so-mirror`. Once completed with the grid setup, you can later tag all your VMs, whose traffic you want monitored, with the same `so-mirror` tag. Under Select collector destination, choose the front end forwarding rule that was created during the Load Balancer setup earlier. @@ -127,7 +127,7 @@ To configure a Security Onion instance (repeat for each node in a distributed Gr - Access the Google Cloud Marketplace at . - Ensure you have a means of authenticating to VM instances over SSH. One method to authenticate is via a project-wide SSH key, which can be defined in Compute Engine -> Metadata -> SSH Keys. -- Search the Marketplace for `Security Onion` and Launch the latest version of the Security Onion 2 official VM image. This may require clicking the "Get Started" button. +- Search the Marketplace for `Security Onion` and Launch the latest version of the Security Onion official VM image. This may require clicking the "Get Started" button. - Choose the appropriate machine type based on the desired hardware requirements. For assistance on determining resource requirements please review the Requirements section above. - Under the Networking interfaces section, expand the pre-added Network interface and select the Security Onion VPC network and desired subnet. External ephemeral IP is sufficient, unless you are planning to use a VPN to access the Security Onion Console, in which case no external ephemeral IP is necessary. Using a VPN is recommended, but setup of a VPN in GCP is out of scope of this guide. - (Distributed "Sensor" node or Single-Node Grid only) Add a second Network interface and select the monitoring VPC network, and the appropriate subnet. No external ephemeral IP is necessary for this interface. @@ -208,9 +208,9 @@ Location: Remote Location: Remote Location: Googe Location: G 192.168.33.13 192.168.33.10 10.55.1.10 10.55.1.20 ``` -In order to add the Remote Network Sensor Node to the Grid, you would have to add `10.55.1.10` to the `sensor` firewall hostgroup. +In order to add the Remote Network Sensor Node to the grid, you would have to add `10.55.1.10` to the `sensor` firewall hostgroup. -This change can be done in the SOC Configuration screen. Then, either wait up to 15 minutes for the scheduled configuration sync to run, or force a synchronization immediately via the SOC Configuration Options. Once the firewall hostgroup configuration has been synchronized your Manager will be ready for remote minions to start connecting. +This change can be done in the SOC Configuration screen. [Auto State Apply](salt.md#auto-state-apply) should apply the new firewall hostgroup configuration within a few minutes, after which your Manager will be ready for remote minions to start connecting. ## Verifying Traffic Mirroring @@ -225,4 +225,4 @@ While that is running, in another terminal, SSH into this new test VM and run a Login to Security Onion and verify that the traffic also appears in the Hunt user interface. -Delete the temporary test VM instance when the verification is completed. \ No newline at end of file +Delete the temporary test VM instance when the verification is completed. diff --git a/docs/connect-api.md b/docs/connect-api.md index 6574193d..715f8512 100644 --- a/docs/connect-api.md +++ b/docs/connect-api.md @@ -1,24 +1,34 @@ -# Connect API +# Security Onion API !!! NOTE This is an enterprise-level feature of Security Onion. Contact Security Onion Solutions, LLC via our website at for more information about purchasing a Security Onion Pro license to enable this feature. -The Security Onion Connect API allows other servers to integrate with Security Onion, and access the same functionality that the Security Onion Console user-interface provides. Access to the Connect API is permitted through API Clients, which can be created by SOC administrators via the SOC UI -> Administration -> API Clients screen. +The Security Onion API allows other servers to integrate with Security Onion, and access the same functionality that the Security Onion Console user-interface provides. Access to the Security Onion API is permitted through API Clients, which can be created by SOC administrators via the SOC UI -> Administration -> API Clients screen. -The Connect API currently provides functionality exposed by the Security Onion Console server. It does not provide full access to third-party applications included with the Security Onion platform. Specifically, while you can read events from Elasticsearch, you cannot manipulate Kibana settings via the Security Onion Connect API, unless those settings are already exposed via the SOC Configuration system. +The Security Onion API currently provides functionality exposed by the Security Onion Console server. It does not provide full access to third-party applications included with the Security Onion platform. Specifically, while you can read events from Elasticsearch, you cannot manipulate Kibana settings via the Security Onion API, unless those settings are already exposed via the SOC Configuration system. -## Enabling Connect API +## API Reference + +!!! WARNING + + New releases of Security Onion may contain additional fields in API responses. Consequently, it is important that the API output be properly parsed by official libraries that can handle these scenarios. Using custom parsing of API outputs may lead to upgrade-related malfunctions. + +An interactive API view is available for browser-based viewing: Interactive API + +In order to connect to the API, you will need to follow the steps below. + +## Enabling Security Onion API By default, newly setup grids will not be configured for API client access. To enable API client access, the following steps must be taken: -1. A license key must be applied to the Grid. The license key must include the API feature. +1. A license key must be applied to the grid. The license key must include the API feature. 2. The Hydra feature must be enabled via the `hydra > enabled` setting in the Configuration screen. -3. Synchronize the Grid to apply the license key and configuration changes. This can be done via the Configuration screen options dropdown. +3. Synchronize the grid to apply the license key and configuration changes. This can be done via the Configuration screen options dropdown. ## API Client Credentials -In order to communicate with the Connect API, an API Client must be created. Navigate to the Administration menu using a superuser account. Under the Administration menu click the API Clients menu option. Create a new API client using a short name that reflects the intended usage of this client. Use the Notes field to provide more information, if desired. Upon saving the new client a generated secret will be issued. This client ID and secret pair is needed to authenticate to the Connect API. Protect these credentials using industry best practices. +In order to communicate with the Security Onion API, an API Client must be created. Navigate to the Administration menu using a superuser account. Under the Administration menu click the API Clients menu option. Create a new API client using a short name that reflects the intended usage of this client. Use the Notes field to provide more information, if desired. Upon saving the new client a generated secret will be issued. This client ID and secret pair is needed to authenticate to the Security Onion API. Protect these credentials using industry best practices. ## Authorization / RBAC @@ -45,7 +55,7 @@ curl --cacert ca.crt -X POST -u socl_my_new_client:hwKHspsX2bMuoIs7kGwN https:// Where you will replace: -- `ca.crt` with your manager's certificate authority. If a custom certificate has been applied to your Grid after setup completed, you can access it via the Configuration screen (requires superuser role) from the `nginx > ssl > SSL/TLS Cert File [adv]` config setting, or if using the default generated certificate authority, retrieve the `/etc/pki/ca.crt` certificate file via SSH from the manager node. +- `ca.crt` with your manager's certificate authority. If a custom certificate has been applied to your grid after setup completed, you can access it via the Configuration screen (requires superuser role) from the `nginx > ssl > SSL/TLS Cert File [adv]` config setting, or if using the default generated certificate authority, retrieve the `/etc/pki/ca.crt` certificate file via SSH from the manager node. - `socl_my_new_client` with your client ID (generated by SOC during API client creation) - `hwKHspsX2bMuoIs7kGwN` with your API client's generated secret - `BASE_URL` with your manager's IP or hostname, depending on which option you selected during Security Onion setup @@ -73,11 +83,3 @@ Where the provided bearer token above must be replaced with the access token ext ## Manager of Managers To interact with subgrid data, while still communicating with the primary **Manager of Managers (MoM)** node, include an additional query string parameter on the API URL. The parameter key is `gridId` and the value should be set to the desired subgrid ID. - -## API Reference - -!!! WARNING - - New releases of Security Onion may contain additional fields in API responses. Consequently, it is important that the API output be properly parsed by official libraries that can handle these scenarios. Using custom parsing of API outputs may lead to upgrade-related malfunctions. - -An interactive API view is available: [Interactive API](api/) \ No newline at end of file diff --git a/docs/console.md b/docs/console.md deleted file mode 100644 index 72c10696..00000000 --- a/docs/console.md +++ /dev/null @@ -1,7 +0,0 @@ -# Console - -The current version of Security Onion automatically disables kernel messages in the local console (tty). If you are running an older version of Security Onion and log into the local console, you may see lots of messages from the Linux kernel. To avoid these kernel messages, you have a few options: - -- You can use [SSH](ssh.md) instead of the local console. -- If you really need to use the local console, you can temporarily disable console messages with `sudo dmesg -D`. For more information about dmesg, please see . Also see and . -- Upgrade to the latest version of Security Onion using [soup](soup.md). \ No newline at end of file diff --git a/docs/customizing.md b/docs/customizing.md index 59bf4f71..f2f80fa3 100644 --- a/docs/customizing.md +++ b/docs/customizing.md @@ -1,18 +1,3 @@ -# Customizing for Your Environment +# Customizing Overview This section covers how to customize Security Onion for your environment. - -## Table of Contents - -- [Security Onion Console Customization](security-onion-console-customization.md) -- [nginx](nginx.md) -- [proxy](proxy.md) -- [firewall](firewall.md) -- [email](email.md) -- [NTP](ntp.md) -- [Console](console.md) -- [SSH](ssh.md) -- [hostname](hostname.md) -- [IP](ip.md) -- [DNS](dns.md) -- [URL Base](url-base.md) \ No newline at end of file diff --git a/docs/cyberchef.md b/docs/cyberchef.md index 8192550a..0032b008 100644 --- a/docs/cyberchef.md +++ b/docs/cyberchef.md @@ -1,6 +1,6 @@ # CyberChef -[SOC](security-onion-console.md) includes a link on the sidebar that takes you to CyberChef. +[Security Onion Console](security-onion-console.md) includes a link on the sidebar that takes you to CyberChef. From : @@ -37,4 +37,4 @@ Suppose you are looking at an interesting HTTP file download in [PCAP](pcap.md) !!! NOTE - For more information about CyberChef, please see . \ No newline at end of file + For more information about CyberChef, please see . diff --git a/docs/dashboards.md b/docs/dashboards.md index 2f6254ed..8e18325a 100644 --- a/docs/dashboards.md +++ b/docs/dashboards.md @@ -1,6 +1,6 @@ # Dashboards -[SOC](security-onion-console.md) includes a Dashboards interface which includes an entire set of pre-built dashboards for our standard data types. +[Security Onion Console](security-onion-console.md) includes a Dashboards interface which includes an entire set of pre-built dashboards for our standard data types. ![Image](images/53_dashboards.png) @@ -218,4 +218,4 @@ Or, combined with other segments: ### Sankey Diagram Recursion -There's a known limitation with Sankey diagrams where the diagram is unable to render all data when multiple fields of the diagram contain the same value. This causes a recursion issue. For example, this can occur if using an OQL query of `* | groupby -sankey source.ip destination.ip` and the included events have a specific IP appearing in both the `source.ip` and `destination.ip` fields. SOC will attempt to prevent the recursion issue by omitting any data that introduces recursion. This can result in some diagrams showing partial data on the diagram, and when this occurs the Sankey diagram will have the phrase `(partial)` appended to the title. In rare scenarios, it's possible for the diagram to be completely blank, such as if all data results have the same value in each field. Following the example mentioned above, this could happen if the `source.ip` and `destination.ip` were always equal. \ No newline at end of file +There's a known limitation with Sankey diagrams where the diagram is unable to render all data when multiple fields of the diagram contain the same value. This causes a recursion issue. For example, this can occur if using an OQL query of `* | groupby -sankey source.ip destination.ip` and the included events have a specific IP appearing in both the `source.ip` and `destination.ip` fields. SOC will attempt to prevent the recursion issue by omitting any data that introduces recursion. This can result in some diagrams showing partial data on the diagram, and when this occurs the Sankey diagram will have the phrase `(partial)` appended to the title. In rare scenarios, it's possible for the diagram to be completely blank, such as if all data results have the same value in each field. Following the example mentioned above, this could happen if the `source.ip` and `destination.ip` were always equal. diff --git a/docs/detections.md b/docs/detections.md index c7fd5af3..422e22e5 100644 --- a/docs/detections.md +++ b/docs/detections.md @@ -1,6 +1,6 @@ # Detections -[SOC](security-onion-console.md) includes our Detections interface for managing all of your rules: +[Security Onion Console](security-onion-console.md) includes our Detections interface for managing all of your rules: - [NIDS](nids.md) rules that get loaded into [Suricata](suricata.md) - [Sigma](sigma.md) rules that get loaded into [ElastAlert](elastalert.md) @@ -19,7 +19,7 @@ The upper-right corner shows a count of detections that matched the search query Here is the list of possible status messages and what they mean: - **Pending**: The browser is waiting for the server to send an initial status report. -- **Import Pending**: The import will start once the system stabilizes, usually within twenty minutes. Imports take place only once, after upgrading to Security Onion 2.4.70+. +- **Import Pending**: The import will start once the system stabilizes, usually within twenty minutes. - **Importing**: The previous version of Security Onion's rules are being imported into the new Detections system. This can take an hour or more on some systems. - **Migrating**: Rules will be migrated between Security Onion versions following system upgrades. This can take some time if upgrading from a much older version. - **Migration Failed**: A failure occurred during the migration. The migration will stop on the first error and will not attempt to migrate to newer versions until the issue is resolved. @@ -98,6 +98,10 @@ The TUNING tab allows you to tune the detection. For [NIDS](nids.md) rules, you ![Image](images/60_detection_nids_2_tuning_1.png) +!!! TIP + + NIDS overrides created here are backed up nightly and can be restored with [so-detections-overrides-import](so-detections-overrides-import.md), which is useful when migrating to or rebuilding a manager. + The PLAYBOOKS tab shows any applicable plays for this detection. These playbooks are used for the Guided Analysis tab in [Alerts](alerts.md). !!! WARNING @@ -155,4 +159,4 @@ Suricata/NIDS - All ruleset sources (ETOPEN, ETPRO, custom URL, local directory): UI and disk change once the [Suricata](suricata.md) engine syncs Strelka/YARA - - Git repo (https or disk): UI and disk change once the `SOC` state runs again and the [Strelka](strelka.md) engine syncs \ No newline at end of file + - Git repo (https or disk): UI and disk change once the `SOC` state runs again and the [Strelka](strelka.md) engine syncs diff --git a/docs/directory.md b/docs/directory.md index 40e52837..8253cbf6 100644 --- a/docs/directory.md +++ b/docs/directory.md @@ -2,7 +2,7 @@ ## /opt/so/conf -Applications read their configuration from `/opt/so/conf/`. However, please keep in mind that most config files are managed with [salt](salt.md), so if you manually modify those config files, your changes may be overwritten at the next Salt update. +Applications read their configuration from `/opt/so/conf/`. However, please keep in mind that most config files are managed with [Salt](salt.md), so if you manually modify those config files, your changes may be overwritten at the next Salt update. ## /opt/so/log @@ -14,7 +14,7 @@ Debug logs are stored in `/opt/so/log/`. ## /opt/so/saltstack/local -Custom [salt](salt.md) settings can be added to `/opt/so/saltstack/local/`. +Custom [Salt](salt.md) settings can be added to `/opt/so/saltstack/local/`. ## /nsm diff --git a/docs/dns.md b/docs/dns.md index 190c2905..2c31db29 100644 --- a/docs/dns.md +++ b/docs/dns.md @@ -1,15 +1,3 @@ # DNS -DNS is normally configured during initial setup. If you need to later change your DNS settings, you can use NetworkManager's console utilities, nmtui and nmcli. - -## nmtui - -nmtui is NetworkManager's text-based user interface: - - - -## nmcli - -nmcli is NetworkManager's command-line interface: - - \ No newline at end of file +DNS is normally configured during initial setup. If you need to later change your DNS settings, you can use NetworkManager's console utilities, nmtui (text based user interface) and nmcli (command line inteface). You can learn more at . diff --git a/docs/download.md b/docs/download.md index 2badf5a2..e0e04932 100644 --- a/docs/download.md +++ b/docs/download.md @@ -6,7 +6,7 @@ Before downloading, we highly recommend that you review the [Release Notes](rele **ALWAYS verify the checksum of the ISO image before booting!** This ensures that the ISO image hasn't been tampered with or corrupted during download. If it fails to verify, try downloading again. If it still fails to verify, try downloading from another computer or another network. - Download and verify our ISO image as shown at . + Download and verify our ISO image as shown at . !!! WARNING @@ -14,4 +14,4 @@ Before downloading, we highly recommend that you review the [Release Notes](rele !!! NOTE - If you're going to create a bootable USB from the ISO image, there are many ways to do that. One popular choice that seems to work well for many folks is Balena Etcher which can be downloaded at . \ No newline at end of file + If you're going to create a bootable USB from the ISO image, there are many ways to do that. One popular choice that seems to work well for many folks is Balena Etcher which can be downloaded at . diff --git a/docs/downloads.md b/docs/downloads.md index f15890da..41fcc73c 100644 --- a/docs/downloads.md +++ b/docs/downloads.md @@ -1,6 +1,6 @@ # Downloads -[SOC](security-onion-console.md) includes a Downloads interface that allows you to download the [Elastic Agent](elastic-agent.md) for various operating systems. +[Security Onion Console](security-onion-console.md) includes a Downloads interface that allows you to download the [Elastic Agent](elastic-agent.md) for various operating systems. ![Image](images/78_downloads.png) @@ -10,4 +10,4 @@ !!! NOTE - When installing the Elastic Agent onto remote systems, be sure to allow network access through the [Firewall](firewall.md). \ No newline at end of file + When installing the Elastic Agent onto remote systems, be sure to allow network access through the [Firewall](firewall.md). diff --git a/docs/elastalert.md b/docs/elastalert.md index 0b1df041..fb2711c2 100644 --- a/docs/elastalert.md +++ b/docs/elastalert.md @@ -41,11 +41,11 @@ ElastAlert 2 stores rule status information, such as number of hits, times each ### so-elastalert-create -`so-elastalert-create` is a tool created by [Bryant Treacle](https://github.com/bryant-treacle/so-elastalert-create) that can be used to help ease the pain of ensuring correct syntax and creating ElastAlert rules from scratch. It will walk you through various questions, and eventually output an ElastAlert rule file that you can deploy in your environment to start alerting quickly and easily. +`so-elastalert-create` can be used to help ease the pain of ensuring correct syntax and creating ElastAlert rules from scratch. It will walk you through various questions, and eventually output an ElastAlert rule file that you can deploy in your environment to start alerting quickly and easily. ### so-elastalert-test -`so-elastalert-test` is a wrapper script originally written by Bryant Treacle for ElastAlert's `elastalert-test-rule` tool. The script allows you to test an ElastAlert rule and get results immediately. Simply run `so-elastalert-test`, and follow the prompt(s). +`so-elastalert-test` is a wrapper script for ElastAlert's `elastalert-test-rule` tool. The script allows you to test an ElastAlert rule and get results immediately. Simply run `so-elastalert-test`, and follow the prompt(s). !!! NOTE @@ -83,4 +83,4 @@ You can modify ElastAlert 2 configuration by going to [Administration](administr !!! NOTE - For more information about ElastAlert, please see . \ No newline at end of file + For more information about ElastAlert, please see . diff --git a/docs/elastic-agent.md b/docs/elastic-agent.md index 3d9c5b27..7c98d890 100644 --- a/docs/elastic-agent.md +++ b/docs/elastic-agent.md @@ -7,7 +7,7 @@ Each Security Onion node uses the Elastic Agent to transport logs to [Elasticsea !!! NOTE - In order to receive logs from the Elastic Agent, Security Onion must be running [Logstash](logstash.md). Evaluation Mode and Import Mode do not run [Logstash](logstash.md), so you'll need Standalone or a full Distributed Deployment. In a Distributed Deployment, sensor nodes do not run [Logstash](logstash.md), so you'll need to configure agents to send to your manager or receiver nodes. For more information, please see the [architecture](architecture.md) section. + In order to receive logs from the Elastic Agent, Security Onion must be running [Logstash](logstash.md). Evaluation Mode and Import Mode do not run [Logstash](logstash.md), so you'll need Standalone or a full Distributed Deployment. In a Distributed Deployment, sensor nodes do not run [Logstash](logstash.md), so you'll need to configure agents to send to your manager or receiver nodes. For more information, please see the [Architecture](architecture.md) section. To deploy an Elastic agent to an endpoint, go to the [SOC](security-onion-console.md) [Downloads](downloads.md) page and download the proper Elastic agent for the operating system of that endpoint. @@ -127,4 +127,4 @@ sudo salt-call state.apply elasticfleet.install_agent_grid !!! NOTE - For more information about the Elastic Agent, please see . \ No newline at end of file + For more information about the Elastic Agent, please see . diff --git a/docs/elastic-fleet.md b/docs/elastic-fleet.md index 905655e2..80e003da 100644 --- a/docs/elastic-fleet.md +++ b/docs/elastic-fleet.md @@ -1,6 +1,6 @@ # Elastic Fleet -[SOC](security-onion-console.md) includes a link on the sidebar that takes you to the Fleet page inside [Kibana](kibana.md). +[Security Onion Console](security-onion-console.md) includes a link on the sidebar that takes you to the Fleet page inside [Kibana](kibana.md). Elastic Fleet is pre-configured during Security Onion setup. If you need to make changes to the configuration, you can do so via the Fleet page in [Kibana](kibana.md) as detailed below. @@ -14,7 +14,7 @@ To view agent details, click the `Host` name. To assign the agent to a new policy, unenroll, upgrade the agent, or perform other actions, click the `Actions` menu on the right side of the agent listing and select the appropriate option. -By default, Elastic Agent is installed on every Security Onion Grid node. As a result, all Grid node agents will be enrolled in the `SO-Grid-Nodes` agent policy. +By default, Elastic Agent is installed on every Security Onion Grid node. As a result, all Grid node agents will be enrolled in the `so-grid-nodes` agent policy. !!! WARNING @@ -49,21 +49,21 @@ Agent policies dictate what data each agent will ingest and forward to Elasticse The individual components within each agent policy are called integrations (referred to as `package policies` at the API level), and refer to a specific input and settings pertinent to a data source. -For example, the `SO-Grid-Nodes` agent policy is comprised of the following integrations: +For example, the `so-grid-nodes` agent policy is comprised of the following integrations: -- Elasticsearch-logs (`Elasticsearch` integration) +- elasticsearch-logs (`Elasticsearch` integration) - import-evtx-logs (`Custom Logs` integration) -- import-Suricata-logs (`Custom Logs` integration) -- import-Zeek-logs (`Custom Logs` integration) +- import-suricata-logs (`Custom Logs` integration) +- import-zeek-logs (`Custom Logs` integration) - kratos-logs (`Custom Logs` integration) -- Osquery-Grid-nodes (`Osquery Manager` integration) -- Redis-logs (`Redis` integration) -- Strelka-logs (`Custom Logs` integration) -- Suricata-logs (`Custom Logs` integration) +- osquery-grid-nodes (`Osquery Manager` integration) +- redis-logs (`Redis` integration) +- strelka-logs (`Custom Logs` integration) +- suricata-logs (`Custom Logs` integration) - syslog-tcp-514 (`Custom Logs` integration) - syslog-udp-514 (`Custom Logs` integration) -- system-Grid-nodes (`System` integration) -- Zeek-logs (`Custom Logs` integration) +- system-grid-nodes (`System` integration) +- zeek-logs (`Custom Logs` integration) ## Agent Policies - endpoints-initial @@ -97,10 +97,6 @@ The `Elastic Defend` integration has both free and paid features. By default, on - Network - Process -!!! TIP - - If you are upgrading from 2.4.160 or earlier you will need to manually enable the 'DNS' event collection feature for macOS found under the 'endpoints-initial' policy. - ### Osquery-endpoints (`Osquery Manager` integration) The `Osquery Manager` integration runs Osquery as a daemon on the endpoint and makes the endpoint available for Live or Scheduled queries through the Osquery manager interface in Kibana. @@ -181,10 +177,10 @@ First, go to [Administration](administration.md) --> Configuration --> elasticfl ![Image](images/config-item-elasticfleet.png) -At the top of the page, click the `Options` menu and then enable the `Show advanced settings` option. Then, navigate to elasticfleet --> config --> server --> custom_fqdn and set your custom FQDN. Within 15 minutes, the Grid will apply these new settings and you should see the new FQDNs show up in Elastic Fleet settings. New agent installers will also be regenerated to use this new setting. +At the top of the page, click the `Options` menu and then enable the `Show advanced settings` option. Then, navigate to elasticfleet --> config --> server --> custom_fqdn and set your custom FQDN. Within a few minutes, [Auto State Apply](salt.md#auto-state-apply) will apply these new settings and you should see the new FQDNs show up in Elastic Fleet settings. New agent installers will also be regenerated to use this new setting. ## More Information !!! NOTE - For more information about Fleet, please see . \ No newline at end of file + For more information about Fleet, please see . diff --git a/docs/elasticsearch.md b/docs/elasticsearch.md index 06fe2bec..592bda25 100644 --- a/docs/elasticsearch.md +++ b/docs/elasticsearch.md @@ -65,7 +65,12 @@ You can configure Elasticsearch by going to [Administration](administration.md) ## Parsing -Elasticsearch receives unparsed logs from [Logstash](logstash.md) or [Elastic Agent](elastic-agent.md). Elasticsearch then parses and stores those logs. Parsers are stored in `/opt/so/conf/elasticsearch/ingest/`. Custom ingest parsers can be placed in `/opt/so/saltstack/local/salt/elasticsearch/files/ingest/`. To make these changes take effect, restart Elasticsearch using `so-elasticsearch-restart`. +Elasticsearch receives unparsed logs from [Logstash](logstash.md) or [Elastic Agent](elastic-agent.md). Elasticsearch then parses and stores those logs. Parsers are stored in `/opt/so/conf/elasticsearch/ingest/`. Custom ingest parsers can be placed in `/opt/so/saltstack/local/salt/elasticsearch/files/ingest/`. [Auto State Apply](salt.md#auto-state-apply) picks these up within a few minutes. If you don't want to wait, apply the Elasticsearch state to all nodes running Elasticsearch: + + +``` +sudo salt -C 'I@elasticsearch:enabled:true' state.apply elasticsearch queue=True +``` [Elastic Agent](elastic-agent.md) may pre-parse or act on data before the data reaches Elasticsearch, altering the data stream or index to which it is written, or other characteristics such as the event dataset or other pertinent information. This configuration is maintained in the agent policy or integration configuration in [Elastic Fleet](elastic-fleet.md). @@ -132,7 +137,7 @@ If you get errors like `failed to create query: field expansion for [*] matches ## Shards -Here are a few tips from : +Here are a few tips from : > **TIP**: Avoid having very large shards as this can negatively affect the cluster's ability to recover from failure. There is no fixed limit on how large shards can be, but a shard size of 50GB is often quoted as a limit that has been seen to work for a variety of use-cases. > @@ -224,21 +229,116 @@ After running the command, the index should no longer use replicas and the statu ## Index Management -Elasticsearch indices are managed by both the `so-elasticsearch-indices-delete` utility and Index Lifecycle Management (ILM). +Most Security Onion data is stored in Elasticsearch data streams. Starting with Security Onion 3.2.0, retention can be managed with either Data Stream Lifecycle Management (DLM) or Index Lifecycle Management (ILM). When using ILM, retention is also managed by the `so-elasticsearch-indices-delete` utility. -!!! NOTE - - Check out our Index Lifecycle Management video at [https://youtu.be/Y6HVein7nP8](https://youtu.be/Y6HVein7nP8)! +| Scenario | DLM | ILM | +|---|---|---| +| Fresh 3.2.0 install | Default and recommended | Choose only for advanced needs | +| SOUP'ed grid | Can manually switch to DLM | Default remains on ILM | + +For most deployments, DLM provides sufficient retention management with low configuration overhead. Choose ILM when data should automatically move between data tiers. For example, a distributed grid may use search nodes with SSD or NVMe storage as the 'hot' data tier for recent, frequently searched data, then move older data to warm or cold nodes with lower-cost storage. + +| Requirement | DLM | ILM| +|---|---|---| +| Straightforward time-based retention | ✅ | - | +| Minimal overall configuration | ✅ | ⚠️ Requires additional configuration | +| Move older data to warm or cold tiers | ❌ | ✅ | +| Set shard count or replica count as data ages | ❌ | ✅ | +| Apply lifecycle actions at different data ages | ❌ | ✅ | +| Advanced phase actions | ⚠️ Limited actions supported | ✅ | + +### DLM + +Data Stream Lifecycle Management (DLM) applies a retention period directly to a data stream. Retention here is defined as the time period for which your data is guaranteed to be stored (assuming enough storage is allocated to Elasticsearch). Elasticsearch is allowed at a later time to delete data older than this time period. Retention can be configured on the data stream level or on a global level. + +New Security Onion 3.2.0 grids use DLM by default. DLM is the recommended choice for single-node deployments and most distributed grids because it allows for the most straightforward configuration, eliminating the need to configure more complex ILM policies. Security Onion data streams use a 90-day retention period by default. !!! WARNING - `so-elasticsearch-indices-delete` is primarily designed for single-node deployments (IMPORT, EVAL, and STANDALONE). Running it on a multi-node deployment with one or more search nodes has the possibility of getting into a corner case state where more data is deleted than intended. Because of this, we are disabling this script on multi-node deployments starting in version 2.4.150. If you have a multi-node deployment and haven't yet updated to 2.4.150, then we HIGHLY recommend that you go ahead and manually disable this script. You can find this setting at [Administration](administration.md) --> Configuration --> Elasticsearch --> index_clean. You will also need to ensure that ILM is configured properly to delete indices before disk usage reaches the Elasticsearch watermark setting. Otherwise, Elasticsearch may stop ingesting new data. + Existing grids will remain on ILM. Only fresh installs of Security Onion 3.2.0 will default to using DLM. + +#### View or configure retention method + +The retention method can be set to either DLM or ILM. Changing this value updates all managed index templates & data streams to use the configured retention method. + +1. In SOC, go to **Administration** --> **Configuration** --> **Elasticsearch**. +2. Select **data_retention_method** to view the current method. +3. Select `DLM` to use Data Stream Lifecycle Management, or select `ILM` to use Index Lifecycle Management. + +When the grid synchronizes, selecting DLM enables DLM lifecycle retention for supported Security Onion data streams. Selecting ILM disables DLM lifecycle retention for those streams and returns their retention management to ILM. + +Before switching methods, review your data-retention requirements and available Elasticsearch storage. In deployments where disk-space-based index cleanup is available with ILM, that cleanup is not used with DLM. Make sure the configured DLM retention periods and available storage leave sufficient free space; otherwise, Elasticsearch can stop ingesting data when disk watermarks are reached. + +!!! WARNING + + Before switching from ILM to DLM, verify the current Elasticsearch [node roles configuration](#elasticsearch-node-roles). You'll want to ensure that all Elasticsearch nodes have the `data` or `data_hot` role. As DLM rolls over data streams, they'll be created with `_tier_preferences: data_hot`. + +!!! NOTE + + When using DLM as your retention method, the so-elasticsearch-indices-delete script is disabled. Retention settings for DLM will need to be configured appropriately to maintain storage usage below the configured watermark. + +#### Configure retention setting + +DLM retention can be managed globally and per data stream. Once configured, the index template(s) will be updated and if a data stream currently exists, it will be updated with the newly configured retention period. Data stream retention is updated in place using the `so-elasticsearch-dlm-apply` script. + +To update the global retention period value: + +1. In SOC, go to **Administration** --> **Configuration** --> **Elasticsearch** +2. Select **index_settings** --> **global_overrides** --> **data_stream_lifecycle** --> **data_retention** + +To update data stream specific retention: + +1. In SOC, go to **Administration** --> **Configuration** --> **Elasticsearch** +2. Select **index_settings** +3. Locate the target data stream (for example, so-zeek) + - If the target data stream is not listed, you may need to first update [managed_integrations](third-party-integrations.md#managing-third-party-integration-index-templates) +4. Select **so-zeek** --> **data_stream_lifecycle** --> **data_retention** + +!!! NOTE + The retention period directly affects how frequently a data stream is automatically rolled over. + + > If retention is less than or equal to 1 day, max_age will be 1 hour. + > If retention is less than or equal to 14 days, max_age will be 1 day + > If retention is less than or equal to 90 days, max_age will be 7 days + > If retention is greater than 90 days, max_age will be 30 days + + Tuning the value of [cluster.lifecycle.default.rollover](elasticsearch.md#clusterlifecycledefaultrollover) allows for more control over how frequently a given data stream is rolled over. + +#### DLM advanced configuration + +##### [cluster.lifecycle.default.rollover](https://www.elastic.co/docs/reference/elasticsearch/configuration-reference/data-stream-lifecycle-settings) + +> This property accepts a key value pair formatted string and configures the conditions that would trigger a data stream to rollover when it has lifecycle configured. + +1. In SOC, go to **Administration** --> **Configuration** --> **Elasticsearch** +2. Select **config** --> **cluster** --> **lifecycle** --> **default** --> **rollover** + +##### [data_streams.lifecycle.poll_interval](https://www.elastic.co/docs/reference/elasticsearch/configuration-reference/data-stream-lifecycle-settings) + +> How often Elasticsearch checks what the next action is for all data streams with a built-in lifecycle. + +1. In SOC, go to **Administration** --> **Configuration** --> **Elasticsearch** +2. Select **config** --> **data_streams** --> **lifecycle** --> **poll_interval** + +##### [data_streams.lifecycle.target.merge.policy.merge_factor](https://www.elastic.co/docs/reference/elasticsearch/configuration-reference/data-stream-lifecycle-settings) + +> Data stream lifecycle implements tail merging by updating the Lucene merge policy factor for the target backing index. The merge factor is both the number of segments that should be merged together, and the maximum number of segments that we expect to find on a given tier. + +1. In SOC, go to **Administration** --> **Configuration** --> **Elasticsearch** --> **config** +2. Select **data_streams** --> **lifecycle** --> **target** --> **merge** --> **policy** --> **merge_factor** + +##### [data_streams.lifecycle.target.merge.policy.floor_segment](https://www.elastic.co/docs/reference/elasticsearch/configuration-reference/data-stream-lifecycle-settings) + +> Data stream lifecycle implements tail merging by updating the Lucene merge policy floor segment for the target backing index. This floor segment size is a way to prevent indices from having a long tail of very small segments. + +1. In SOC, go to **Administration** --> **Configuration** --> **Elasticsearch** --> **config** +2. Select **data_streams** --> **lifecycle** --> **target** --> **merge** --> **policy** --> **floor_segment** ### so-elasticsearch-indices-delete `so-elasticsearch-indices-delete` manages size-based deletion of Elasticsearch indices based on the value of the `Elasticsearch.retention.retention_pct` setting. This setting is checked against the total disk space available for `/nsm/elasticsearch` across all nodes in the Elasticsearch cluster. If your indices are using more than `retention_pct`, then `so-elasticsearch-indices-delete` will delete old indices until disk space consumed by indices is back under `retention_pct`. The default value for this setting is `50` percent so that standalone deployments have sufficient space for not only Elasticsearch but also full packet capture and other logs. For distributed deployments with dedicated search nodes where Elasticsearch is main consumer of disk space, you may want to increase this default value. -To modify the `retention_pct` value, first navigate to [Administration](administration.md) --> Configuration. At the top of the page, click the `Options` menu and then enable the `Show advanced settings` option. Then navigate to Elasticsearch --> retention --> retention_pct. Once you make the change and save it, the new setting will take effect at the next 15 minute interval. If you would like to make the change immediately, you can click the `SYNCHRONIZE Grid` button under the `Options` menu at the top of the page. +To modify the `retention_pct` value, first navigate to [Administration](administration.md) --> Configuration. At the top of the page, click the `Options` menu and then enable the `Show advanced settings` option. Then navigate to Elasticsearch --> retention --> retention_pct. Once you make the change and save it, [Auto State Apply](salt.md#auto-state-apply) should apply the new setting within a few minutes. ### ILM @@ -336,4 +436,4 @@ sudo docker logs so-elasticsearch !!! NOTE For more information about Elasticsearch, please see: - \ No newline at end of file + diff --git a/docs/endgame.md b/docs/endgame.md deleted file mode 100644 index bdd219e4..00000000 --- a/docs/endgame.md +++ /dev/null @@ -1,26 +0,0 @@ -# Endgame - -!!! WARNING - - Endgame support has not been tested yet! - -You can ingest Endgame data by following the steps below. - -!!! NOTE - - Please keep in mind that we currently use the `*:endgame-*` index pattern for Endgame data. This means the data will not be visible using the normal Security Onion dashboards/index pattern in Kibana. However, Endgame data will be viewable and aggregatable using Hunt and Elastic Security. - -## Configuration - -To configure Endgame ingestion during setup, ensure the `ENDGAMEHOST` variable is set to the IP address of the Endgame SMP that you want to send data from: - - -``` -sudo ENDGAMEHOST=192.168.1.100 ./so-setup-network -``` - -This will open the Security Onion host-based firewall for access from the SMP to Security Onion on TCP port 3765. - -## Pivot to Endgame Console - -If Endgame support is enabled, then [Dashboards](dashboards.md) and [Hunt](hunt.md) will have an `Endgame` action on the Actions menu. Clicking that action will pivot to Endgame Console based on the `agent.id` field. \ No newline at end of file diff --git a/docs/eol.md b/docs/eol.md index 053bc914..198606c1 100644 --- a/docs/eol.md +++ b/docs/eol.md @@ -2,17 +2,19 @@ This page lists End Of Life (EOL) dates for older versions of Security Onion and older components. -- Security Onion 2.3 reached EOL on April 6, 2024 (please migrate to Security Onion 2.4): +- Security Onion 2.4 reaches EOL on October 1, 2026 (please migrate to Security Onion 3) + +- Security Onion 2.3 reached EOL on April 6, 2024: -- Ubuntu 18.04 reached End of Ubuntu Standard Support in April 2023: +- Ubuntu 18.04 reached End of Ubuntu Standard Support in April 2023: -- TheHive 3 reached EOL on December 31, 2021. TheHive and Cortex were fully removed from Security Onion in Security Onion 2.3.120: +- TheHive 3 reached EOL on December 31, 2021. TheHive and Cortex were fully removed from Security Onion in Security Onion 2.3.120: -- Security Onion 16.04 reached EOL on April 16, 2021: +- Security Onion 16.04 reached EOL on April 16, 2021: -- Security Onion 14.04 reached EOL on November 30, 2018: - \ No newline at end of file +- Security Onion 14.04 reached EOL on November 30, 2018: + diff --git a/docs/faq.md b/docs/faq.md index 51368ef6..7c16a794 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -87,7 +87,7 @@ No, Security Onion does not support blocking traffic. Most organizations have so ### Where can I read more about the tools contained within Security Onion? -Please see the [Tools](tools.md) section. +Please see the [Software Bill of Materials](software-bill-of-materials.md) section. ### What's the directory structure of `/nsm`? diff --git a/docs/firewall.md b/docs/firewall.md index 2593cba2..ce818fd5 100644 --- a/docs/firewall.md +++ b/docs/firewall.md @@ -89,7 +89,7 @@ sudo iptables -nvL !!! WARNING - You can use this command to view the iptables configuration, but please do not modify the firewall manually using iptables as it is managed by [salt](salt.md). You should only make changes via the Configuration screen as shown above. + You can use this command to view the iptables configuration, but please do not modify the firewall manually using iptables as it is managed by [Salt](salt.md). You should only make changes via the Configuration screen as shown above. ## Port Groups @@ -103,7 +103,7 @@ Host groups are similar to port groups but for storing lists of hosts that will The firewall state is designed with the idea of creating port groups and host groups, each with their own alias or name, and associating the two in order to create an allow rule. A node that has a port group and host group association assigned to it will allow those hosts to connect to those ports on that node. -The default allow rules for each node are defined by its role (manager, searchnode, sensor, heavynode, etc) in the Grid. Host groups and port groups can be created or modified from the manager node by going to [Administration](administration.md) --> Configuration --> firewall --> hostgroups. When setup is run on a new node, it will ask the manager to add itself to the appropriate host groups. All node types are added to the minion host group to allow [salt](salt.md) communication. If you were to add a search node, you would see its IP appear in both the `minion` and the `search_node` host groups. +The default allow rules for each node are defined by its role (manager, searchnode, sensor, heavynode, etc) in the grid. Host groups and port groups can be created or modified from the manager node by going to [Administration](administration.md) --> Configuration --> firewall --> hostgroups. When setup is run on a new node, it will ask the manager to add itself to the appropriate host groups. All node types are added to the minion host group to allow [Salt](salt.md) communication. If you were to add a search node, you would see its IP appear in both the `minion` and the `search_node` host groups. ## Advanced Firewall Config @@ -116,7 +116,7 @@ The analyst hostgroup is allowed access to the nginx ports which are 80 and 443 - At the top of the page, click the `Options` menu and then enable the `Show advanced settings` option. - On the left side, go to `firewall`, select `portgroups`, locate the `nginx` portgroup, and then select `tcp`. - On the right side, select the manager node, specify your custom port to be added, and then click the checkmark to save the value. -- If you would like to apply the rules immediately, click the `SYNCHRONIZE Grid` button under the `Options` menu at the top of the page. +- If you would like to apply the rules immediately, click the `SYNCHRONIZE GRID` button under the `Options` menu at the top of the page. ### Creating a custom host group with a custom port group diff --git a/docs/first-time-users.md b/docs/first-time-users.md index e2313da0..cd49e86d 100644 --- a/docs/first-time-users.md +++ b/docs/first-time-users.md @@ -4,7 +4,7 @@ Welcome, first time users! You're going to be peeling back the layers of your ne First, please note that Security Onion only supports x86-64 architecture (standard Intel or AMD 64-bit processors). If you don't have an x86-64 box available, then one option may be to run Security Onion in the cloud. For more information, please see the [Amazon Cloud](cloud-amazon.md), [Azure Cloud](cloud-azure.md), and [Google Cloud](cloud-google.md) sections. -Otherwise, if you have an x86-64 box for your Security Onion IMPORT installation, then check to make sure it meets the MINIMUM hardware requirements of 4GB RAM, 2 CPU cores, and 200GB of storage. If you will be installing Security Onion in a virtual machine, then the VM will need those specs at minimum and the host machine will have higher hardware requirements since it will be running the host operating system and possibly other VMs or apps. For more information about virtualization, please see the [VMware](vmware.md), [VirtualBox](virtualbox.md), and [Proxmox](proxmox.md) sections. Once you've verified that you have an appropriate installation target, you can proceed to download our ISO image as shown in the [Download](download.md) section and then install the ISO image as shown in the [Installation](installation.md) section. +Otherwise, if you have an x86-64 box for your Security Onion IMPORT installation, then check to make sure it meets the MINIMUM hardware requirements of 4GB RAM, 2 CPU cores, and 100GB of storage. If you will be installing Security Onion in a virtual machine, then the VM will need those specs at minimum and the host machine will have higher hardware requirements since it will be running the host operating system and possibly other VMs or apps. For more information about virtualization, please see the [VMware](vmware.md), [VirtualBox](virtualbox.md), and [Proxmox](proxmox.md) sections. Once you've verified that you have an appropriate installation target, you can proceed to download our ISO image as shown in the [Download](download.md) section and then install the ISO image as shown in the [Installation](installation.md) section. Once you have Security Onion installed either in the cloud or on-prem, you can configure for IMPORT as shown below (also see the [Configuration](configuration.md) section). diff --git a/docs/full-packet-capture.md b/docs/full-packet-capture.md index daa4eb3f..9671d08c 100644 --- a/docs/full-packet-capture.md +++ b/docs/full-packet-capture.md @@ -10,7 +10,7 @@ You can access full packet capture via the [PCAP](pcap.md) interface: ![Image](images/65_pcap_details.png) -[Alerts](alerts.md), [Dashboards](dashboards.md), [Hunt](hunt.md), and [Kibana](kibana.md) allow you to easily pivot to the [PCAP](pcap.md) interface. +[Alerts](alerts.md), [Dashboards](dashboards.md), and [Hunt](hunt.md) allow you to easily pivot to the [PCAP](pcap.md) interface. ## Configuration @@ -30,9 +30,10 @@ By default, Suricata writes all network traffic to PCAP. If you would like to li Here are some other PCAP configuration options that can be found at [Administration](administration.md) --> Configuration --> Suricata -> config -> pcap. Some settings are considered advanced settings so you will only see them if you enable the `Show advanced settings` option. +- `enabled`: Click the slider to enable or disable Suricata packet capture. - `compression`: Set to `none` to disable compression. Set to `lz4` to enable lz4 compression but note that this requires more CPU cycles. - `lz4-level`: lz4 compression level of PCAP files. Set to `0` for no compression. Set to `16` for maximum compression. -- `maxsize`: Maximum size in GB for total disk usage of all PCAP files written by Suricata. If you originally installed version 2.4.60 or newer, then this value should have been set based on a percentage of your disk space. If you originally installed a version older than 2.4.60, then this value should have been set to `25` by default. You may need to adjust this value based on your disk space and desired PCAP retention. +- `maxsize`: Maximum size in GB for total disk usage of all PCAP files written by Suricata. You may need to adjust this value based on your disk space and desired PCAP retention. - `filesize`: Maximum file size for individual PCAP files written by Suricata. Increasing this number could improve write performance at the expense of PCAP retrieval time. - `use-stream-depth`: Set to `no` to ignore the stream depth and capture the entire flow. Set to `yes` to truncate the flow based on the stream depth. @@ -46,4 +47,4 @@ Diagnostic logging for Suricata can be found at `/opt/so/log/suricata/suricata.l ``` sudo docker logs so-suricata -``` \ No newline at end of file +``` diff --git a/docs/getting-started.md b/docs/getting-started.md index 7939fabe..99ef35c1 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,4 +1,4 @@ -# Getting Started +# Getting Started Overview If you're ready to get started with Security Onion, you may have questions like: @@ -41,22 +41,3 @@ The [Configuration](configuration.md) section covers many different use cases. **Is there anything I need to do after configuration?** See the [Post Installation](post-installation.md) section. - -## Table of Contents - -- [Best Practices](best-practices.md) -- [Use Cases](use-cases.md) -- [Architecture](architecture.md) -- [Hardware](hardware.md) -- [Download](download.md) -- [VMware](vmware.md) -- [VirtualBox](virtualbox.md) -- [Proxmox](proxmox.md) -- [Trouble Booting](trouble-booting.md) -- [Airgap](airgap.md) -- [Installation](installation.md) -- [Amazon Cloud](cloud-amazon.md) -- [Azure Cloud](cloud-azure.md) -- [Google Cloud](cloud-google.md) -- [Configuration](configuration.md) -- [Post Installation](post-installation.md) \ No newline at end of file diff --git a/docs/grid.md b/docs/grid.md index 0d6c88f2..47afe764 100644 --- a/docs/grid.md +++ b/docs/grid.md @@ -1,10 +1,10 @@ # Grid -[Security Onion Console](security-onion-console.md) includes a Grid interface which allows you to quickly check the status of all nodes in your Grid. +[Security Onion Console](security-onion-console.md) includes a grid interface which allows you to quickly check the status of all nodes in your grid. ![Image](images/39_grid.png) -Starting at the top of the page, there is a `Grid EPS` value in the upper-right corner that shows the sum of all `Consumption EPS` measurements in the entire Grid. Below that you will find a list of all nodes in your Grid. +Starting at the top of the page, there is a `Grid EPS` value in the upper-right corner that shows the sum of all `Consumption EPS` measurements in the entire Grid. Below that you will find a list of all nodes in your grid. !!! WARNING @@ -62,7 +62,7 @@ The `Last Heard From` field shows the last time that the node checked-in with th ### Age -The `Age` field shows how long the node has been part of the Grid and is based on the `Date Created` value. +The `Age` field shows how long the node has been part of the grid and is based on the `Date Created` value. ### OS Uptime @@ -72,7 +72,7 @@ If the node needs to be restarted to apply kernel updates then a message will ap ### Last Synchronized -The `Last Synchronized` field shows how long ago the node was synchronized to the manager node. This is equivalent to the last Salt highstate run. Knowing this value can be helpful when making configuration changes to the Grid and determining whether a specific node has received those changes. +The `Last Synchronized` field shows how long ago the node was synchronized to the manager node. This is equivalent to the last Salt highstate run. Knowing this value can be helpful when making configuration changes to the grid and determining whether a specific node has received those changes. ### Process Status @@ -80,7 +80,7 @@ If the `Process Status` field shows `Fault`, you can check the other status indi ### Connection Status -The `Connection Status` field shows whether or not the node is currently connected to the Grid. +The `Connection Status` field shows whether or not the node is currently connected to the grid. ### Elasticsearch Status @@ -138,7 +138,7 @@ The `Elastic Storage Used` field shows the total gigabytes used by [Elasticsearc ### InfluxDB Storage Used -The `InfluxDB Storage Used` field shows the total gigabytes used by [InfluxDB](influxdb.md) to store the current and historic metric data collected from all nodes in the Grid. +The `InfluxDB Storage Used` field shows the total gigabytes used by [InfluxDB](influxdb.md) to store the current and historic metric data collected from all nodes in the grid. ### PCAP Retention @@ -188,7 +188,7 @@ There are a few icons in the lower left of the `Node Status` section depending o ![Image](images/40_upload.png) -- The reboot button allows for remotely rebooting a Grid node. This may be necessary when scheduled OS/kernel updates are automatically applied and require a restart to take effect. Review the notes on the confirmation dialog thoroughly before confirming a reboot. Rebooting a manager node will likely cause the Security Onion Console web interface to become temporarily unavailable. +- The reboot button allows for remotely rebooting a grid node. This may be necessary when scheduled OS/kernel updates are automatically applied and require a restart to take effect. Review the notes on the confirmation dialog thoroughly before confirming a reboot. Rebooting a manager node will likely cause the Security Onion Console web interface to become temporarily unavailable. - Clicking the question mark button takes you to this help document. @@ -208,4 +208,4 @@ If a node is running on an official Security Onion Solutions appliance then the !!! NOTE - You can manage Grid members and Grid configuration in the [Administration](administration.md) section. \ No newline at end of file + You can manage Grid members and Grid configuration in the [Administration](administration.md) section. diff --git a/docs/hardware.md b/docs/hardware.md index b23e4676..15238b39 100644 --- a/docs/hardware.md +++ b/docs/hardware.md @@ -14,7 +14,8 @@ Security Onion only supports x86-64 architecture (standard Intel or AMD 64-bit p | Node Type | CPU cores | RAM | Storage | NICs | |-----------------|-----------|------|---------|------| -| Import | 2 | 4GB | 50GB | 1 | +| Desktop | 2 | 4GB | 50GB | 1 | +| Import | 2 | 4GB | 100GB | 1 | | Eval | 4 | 8GB | 200GB | 2 | | Standalone | 4 | 24GB | 200GB | 2 | | Manager | 4 | 16GB | 200GB | 1 | @@ -32,13 +33,11 @@ Security Onion only supports x86-64 architecture (standard Intel or AMD 64-bit p ## Import -An Import installation runs the minimal processes required to import PCAP or EVTX files and view the results. As such, it has the lowest hardware requirements as shown in the table above. You can read more about Import in the [First Time Users](first-time-users.md) section. +An Import installation runs the minimal processes required to import PCAP or EVTX files and view the results. You can read more about Import in the [First Time Users](first-time-users.md) section. ## Eval -An Eval installation runs the minimal processes required for a single machine to sniff live network traffic from a TAP or SPAN port and view the results. Therefore, its hardware requirements are higher than Import as shown in the table above. Eval is designed for temporary installations or homelab installations on a budget. Unlike a full Standalone installation, Evaluation is NOT designed for production usage. - -In order to minimize RAM usage, Eval does not run [Logstash](logstash.md) or [Redis](redis.md) at all. Also, Eval uses [Suricata](suricata.md) for writing full packet capture to disk. +An Eval installation runs the minimal processes required for a single machine to sniff live network traffic from a TAP or SPAN port and view the results. Therefore, its hardware requirements are higher than Import as shown in the table above. Eval is designed for temporary installations or homelab installations on a budget. Unlike a full Standalone installation, Evaluation is NOT designed for production usage. In order to minimize RAM usage, Eval does not run [Logstash](logstash.md) or [Redis](redis.md) at all. ## Production Deployments @@ -171,11 +170,11 @@ The following RAM estimates are a rough guideline and assume that you're going t - If you're deploying Security Onion in production on a small network (100Mbps or less), you should plan on 16GB RAM or more. Again, more is obviously better! - If you're deploying Security Onion in production to a medium network (100Mbps - 1000Mbps), you should plan on 16GB - 128GB RAM or more. - If you're deploying Security Onion in production to a large network (1000Mbps - 10Gbps), you should plan on 128GB - 256GB RAM or more. -- If you're buying a new server, go ahead and max out the RAM (it's cheap!). As always, more is obviously better! +- If you're buying a new server, consider maxing out the RAM if possible. ### Storage -Sensors that have full packet capture enabled need LOTS of storage. For example, suppose you are monitoring a link that averages 50Mbps, here are some quick calculations: 50Mb/s = 6.25 MB/s = 375 MB/minute = 22,500 MB/hour = 540,000 MB/day. So you're going to need about 540GB for one day's worth of pcaps (multiply this by the number of days of PCAP you want to keep). The more disk space you have, the more PCAP retention you'll have for doing investigations after the fact. Disk is cheap, get all you can! +Sensors that have full packet capture enabled need LOTS of storage. For example, suppose you are monitoring a link that averages 50Mbps, here are some quick calculations: 50Mb/s = 6.25 MB/s = 375 MB/minute = 22,500 MB/hour = 540,000 MB/day. So you're going to need about 540GB for one day's worth of pcaps (multiply this by the number of days of PCAP you want to keep). The more disk space you have, the more PCAP retention you'll have for doing investigations after the fact. ### Packets @@ -185,7 +184,6 @@ Inexpensive TAP/SPAN options (listed alphabetically): - [Dualcomm](https://www.dualcomm.com/collections/network-tap) - [Midbit SharkTap](https://www.midbittech.com) -- [Mikrotik](https://mikrotik.com/product/rb260gs) - [Netgear GS105Ev2](https://www.netgear.com/support/product/gs105ev2) Enterprise TAP options (listed alphabetically): @@ -202,4 +200,4 @@ Enterprise TAP options (listed alphabetically): !!! NOTE - For large networks and/or deployments, please also see [https://github.com/pevma/SEPTun](https://github.com/pevma/SEPTun). \ No newline at end of file + For large networks and/or deployments, please also see [https://github.com/pevma/SEPTun](https://github.com/pevma/SEPTun). diff --git a/docs/help-wanted.md b/docs/help-wanted.md index aa0c08c2..69f12199 100644 --- a/docs/help-wanted.md +++ b/docs/help-wanted.md @@ -14,9 +14,7 @@ If you'd like help out other Security Onion users, please join the forum and sta ## Documentation Team -If you find that some information in our Documentation is incorrect or lacking, please feel free to submit Pull Requests via GitHub! - - +If you find that some information in our Documentation is incorrect or lacking, please feel free to submit Pull Requests to the `3/dev` branch of the `docs` repo at . ## Core Development @@ -53,4 +51,5 @@ The following folks have made significant contributions to Security Onion over t - Jon Schipp - Brad Shoop - Bryant Treacle -- William Wernert \ No newline at end of file +- William Wernert +- Matthew Wright diff --git a/docs/help.md b/docs/help.md index 55dc5c7b..f943dc77 100644 --- a/docs/help.md +++ b/docs/help.md @@ -1,11 +1,11 @@ -# Help +# Help Overview Having problems? Try the suggestions below. - Have you run [soup](soup.md) to ensure that you're on the latest version? - Check the [FAQ](faq.md). - Search the [Community Support](community-support.md) forum. -- Search the documentation and support forums of the [tools](tools.md) contained within Security Onion. +- Search the documentation and support forums of the [tools](software-bill-of-materials.md) contained within Security Onion. - Check log files in `/opt/so/log/` or other locations for any errors or possible clues: - Setup `/root/sosetup.log` @@ -19,12 +19,3 @@ Having problems? Try the suggestions below. - Are you able to duplicate the problem on a fresh Security Onion installation? - Check the [Known Issues](https://github.com/security-onion-solutions/securityonion/issues) to see if this is a known issue that we are working on. - If all else fails, please feel free to reach out for [support](support.md). - -## Table of Contents - -- [FAQ](faq.md) -- [Directory](directory.md) -- [Tools](tools.md) -- [Support](support.md) -- [Community Support](community-support.md) -- [Help Wanted](help-wanted.md) diff --git a/docs/host-visibility.md b/docs/host-visibility.md index 2ae287a5..8b905059 100644 --- a/docs/host-visibility.md +++ b/docs/host-visibility.md @@ -1,4 +1,4 @@ -# Host Visibility +# Host Visibility Overview More and more of our network traffic is encrypted these days and that's a good thing for privacy but it's somewhat of a blind spot for us as defenders. Host visibility can help fill in those blind spots. You can send host logs to Security Onion via your choice of either [Elastic Agent](elastic-agent.md) or [Syslog](syslog.md): @@ -6,9 +6,3 @@ More and more of our network traffic is encrypted these days and that's a good t - Choose [Syslog](syslog.md) if you can't install an agent but the device supports sending standard syslog. Examples include firewalls, switches, routers, and other network devices. For Windows endpoints, you can optionally augment the standard Windows logging with [Sysmon](sysmon.md). - -## Table of Contents - -- [Elastic Agent](elastic-agent.md) -- [Syslog](syslog.md) -- [Sysmon](sysmon.md) \ No newline at end of file diff --git a/docs/hostname.md b/docs/hostname.md index 80cc10db..5ef69d16 100644 --- a/docs/hostname.md +++ b/docs/hostname.md @@ -1,3 +1,3 @@ # Hostname -Setup generates certificates based on the hostname and we do not support changing the hostname after Setup. Please make sure that your hostname is correct during installation. \ No newline at end of file +Setup generates certificates based on the hostname and we do not support changing the hostname after Setup. Please make sure that your hostname is correct during installation as mentioned in the [Best Practices](best-practices.md) section. diff --git a/docs/hunt.md b/docs/hunt.md index 3d0f08a0..5573e84e 100644 --- a/docs/hunt.md +++ b/docs/hunt.md @@ -1,9 +1,9 @@ # Hunt -[SOC](security-onion-console.md) includes a Hunt interface which is similar to our [Dashboards](dashboards.md) interface but is tuned more for threat hunting. +[Security Onion Console](security-onion-console.md) includes a Hunt interface which is similar to our [Dashboards](dashboards.md) interface but is tuned more for threat hunting. ![Image](images/56_hunt.png) The main difference between Hunt and [Dashboards](dashboards.md) is that Hunt's default queries are more focused than the overview queries in [Dashboards](dashboards.md). A second difference is that most of the default [Dashboards](dashboards.md) queries display a separate table for each aggregated field, whereas many of the default queries in Hunt aggregate multiple fields in a single table which can be beneficial when hunting for more obscure activity. -Other than these two differences, Hunt and [Dashboards](dashboards.md) are very similar, so for more information please see the [Dashboards](dashboards.md) section. \ No newline at end of file +Other than these two differences, Hunt and [Dashboards](dashboards.md) are very similar, so for more information please see the [Dashboards](dashboards.md) section. diff --git a/docs/hypervisor.md b/docs/hypervisor.md index 416ece7d..edeeaf7a 100644 --- a/docs/hypervisor.md +++ b/docs/hypervisor.md @@ -1,6 +1,6 @@ # Hypervisor -Starting with Security Onion version 2.4.170, Security Onion Pro users can create a hypervisor node that can run virtualized instances of Security Onion. If you have eligible machines with extra horsepower, you can use this feature to spin up additional Security Onion virtual machines (VMs) to take advantage of that extra power. This supports most major Security Onion node types and is especially helpful if you want to optimize your hardware's potential and expand your Elastic performance and retention. Contact your account manager to see if your hardware is supported. +Security Onion Pro customers can create a hypervisor node that can run virtualized instances of Security Onion. If you have eligible machines with extra horsepower, you can use this feature to spin up additional Security Onion virtual machines (VMs) to take advantage of that extra power. This supports most major Security Onion node types and is especially helpful if you want to optimize your hardware's potential and expand your Elastic performance and retention. Contact your account manager to see if your hardware is supported. !!! NOTE @@ -29,7 +29,7 @@ The following resources will be reserved for the host machine and will be subtra ## Airgap -If you are in an [airgap](airgap.md) environment, you will need to perform these steps prior to accepting the new hypervisor node in SOC Grid Members: +If you are in an [Airgap](airgap.md) environment, you will need to perform these steps prior to accepting the new hypervisor node in SOC Grid Members: 1. Download the Oracle 9 Qcow2 image from @@ -37,7 +37,7 @@ If you are in an [airgap](airgap.md) environment, you will need to perform these ## Adding a Manager + Hypervisor -Starting in 2.4.180, Security Onion Pro users can create a manager node that also has hypervisor capabilities. This node type is called a `managerhype`. +Manager nodes can have hypervisor capabilities. This node type is called a `managerhype`. Install a new node, select the `DISTRIBUTED` deployment option, choose `New Deployment`, and then select the `Managerhype` option: @@ -57,7 +57,7 @@ Install a new node, select the `DISTRIBUTED` deployment option, choose `Existing The manager will need to be able to connect to the hypervisor node by name so it will either need a DNS entry or you can manually add an entry in /etc/hosts on the manager. -Once the new hypervisor node has been accepted into the Grid, go to SOC Configuration, click the Options menu, enable advanced settings, and then navigate to `hypervisor` settings. It should look like this: +Once the new hypervisor node has been accepted into the grid, go to SOC Configuration, click the Options menu, enable advanced settings, and then navigate to `hypervisor` settings. It should look like this: ![Image](images/hypervisor/hyper-1.png) @@ -79,7 +79,7 @@ The vast majority of data, for all node types, is stored in /nsm/. For a VM, the #. **Virtual disk** - Added in 2.4.190, a virtual disk is created based on the size specified by the user in the SOC Grid Configuration and the space is pre-allocated on the hypervisor. The disk image file is not removed when the VM is deleted. A user may decide to leave this data around for a while, or delete it manually from the hypervisor where it is stored under `/nsm/libvirt/volumes`. + A virtual disk is created based on the size specified by the user in the SOC Grid Configuration and the space is pre-allocated on the hypervisor. The disk image file is not removed when the VM is deleted. A user may decide to leave this data around for a while, or delete it manually from the hypervisor where it is stored under `/nsm/libvirt/volumes`. !!! NOTE @@ -175,4 +175,4 @@ You will then need to refresh your web browser to see that CPU, memory, and free !!! WARNING - If you delete a VM and attempt to immediately create a new VM prior to the backend releasing the hardware, then you will not be able to pass through the hardware that was previously used by the deleted VM. \ No newline at end of file + If you delete a VM and attempt to immediately create a new VM prior to the backend releasing the hardware, then you will not be able to pass through the hardware that was previously used by the deleted VM. diff --git a/docs/idh.md b/docs/idh.md index d81d699e..1aaeeee7 100644 --- a/docs/idh.md +++ b/docs/idh.md @@ -22,7 +22,7 @@ IDH nodes are dedicated to just being IDH nodes and cannot run any other service - Select the `Existing Deployment` option. - Select the `IDH` option. - You can optionally prevent the IDH services from listening on the management interface. -- Once Setup is complete and the IDH node is fully joined to the Grid, you can do additional configuration by going to [Administration](administration.md) --> Configuration --> IDH. +- Once Setup is complete and the IDH node is fully joined to the grid, you can do additional configuration by going to [Administration](administration.md) --> Configuration --> IDH. - After configuration is complete, connections to honeypot services will result in `SO IDH` alerts that can be seen in [Alerts](alerts.md). ## Technical Background @@ -86,8 +86,8 @@ For example, suppose that we already have the HTTP service running but we want t - At the top of the page, click the `Options` menu and enable the `Show advanced settings` option. - On the left side, navigate to IDH --> opencanary --> config --> http_x_port. - On the right side, change the port value and then click the checkmark to save the change. -- At the top of the page, click the `SYNCHRONIZE Grid` button under the `Options` menu. +- At the top of the page, click the `SYNCHRONIZE GRID` button under the `Options` menu. ## Activating Additional Network Interfaces -If you want to activate additional network interfaces after joining your IDH node to your Grid, you can do so using standard Linux networking tools like nmtui. You can read more about nmtui at . \ No newline at end of file +If you want to activate additional network interfaces after joining your IDH node to your grid, you can do so using standard Linux networking tools like nmtui. You can read more about nmtui at . diff --git a/docs/images/01_grub.png b/docs/images/01_grub.png index 486b6d58..6581a8cc 100644 Binary files a/docs/images/01_grub.png and b/docs/images/01_grub.png differ diff --git a/docs/images/04_setup_init.png b/docs/images/04_setup_init.png index f1edce32..b138224f 100644 Binary files a/docs/images/04_setup_init.png and b/docs/images/04_setup_init.png differ diff --git a/docs/images/05_setup_option.png b/docs/images/05_setup_option.png index a9649bd8..d9a65af8 100644 Binary files a/docs/images/05_setup_option.png and b/docs/images/05_setup_option.png differ diff --git a/docs/images/06_setup_airgap.png b/docs/images/06_setup_airgap.png index 0a50dfe7..04be2e3a 100644 Binary files a/docs/images/06_setup_airgap.png and b/docs/images/06_setup_airgap.png differ diff --git a/docs/images/06_setup_type.png b/docs/images/06_setup_type.png index 00d578a3..87291c6f 100644 Binary files a/docs/images/06_setup_type.png and b/docs/images/06_setup_type.png differ diff --git a/docs/images/07_setup_license.png b/docs/images/07_setup_license.png index a1dd34e8..ff925af1 100644 Binary files a/docs/images/07_setup_license.png and b/docs/images/07_setup_license.png differ diff --git a/docs/images/08_setup_hostname.png b/docs/images/08_setup_hostname.png index 44304561..a6b43bf1 100644 Binary files a/docs/images/08_setup_hostname.png and b/docs/images/08_setup_hostname.png differ diff --git a/docs/images/09_setup_hostname_conflict.png b/docs/images/09_setup_hostname_conflict.png index edb7bcc1..c0d3ffd4 100644 Binary files a/docs/images/09_setup_hostname_conflict.png and b/docs/images/09_setup_hostname_conflict.png differ diff --git a/docs/images/10_setup_mn_nic.png b/docs/images/10_setup_mn_nic.png index 40f03435..92de17f0 100644 Binary files a/docs/images/10_setup_mn_nic.png and b/docs/images/10_setup_mn_nic.png differ diff --git a/docs/images/11_setup_mn_int.png b/docs/images/11_setup_mn_int.png index 8a68ed0d..3b7bc004 100644 Binary files a/docs/images/11_setup_mn_int.png and b/docs/images/11_setup_mn_int.png differ diff --git a/docs/images/12_setup_cidr.png b/docs/images/12_setup_cidr.png index 02573504..2b5442b7 100644 Binary files a/docs/images/12_setup_cidr.png and b/docs/images/12_setup_cidr.png differ diff --git a/docs/images/13_setup_gateway.png b/docs/images/13_setup_gateway.png index 71d9273d..b56c033b 100644 Binary files a/docs/images/13_setup_gateway.png and b/docs/images/13_setup_gateway.png differ diff --git a/docs/images/14_setup_dns_servers.png b/docs/images/14_setup_dns_servers.png index 319c033b..cd2abd23 100644 Binary files a/docs/images/14_setup_dns_servers.png and b/docs/images/14_setup_dns_servers.png differ diff --git a/docs/images/15_setup_dns_domain.png b/docs/images/15_setup_dns_domain.png index 040fa51b..7d68828f 100644 Binary files a/docs/images/15_setup_dns_domain.png and b/docs/images/15_setup_dns_domain.png differ diff --git a/docs/images/16_setup_docker_range.png b/docs/images/16_setup_docker_range.png index d3cec11d..1d74c3a4 100644 Binary files a/docs/images/16_setup_docker_range.png and b/docs/images/16_setup_docker_range.png differ diff --git a/docs/images/18_setup_direct_proxy.png b/docs/images/18_setup_direct_proxy.png index a91329b2..d3fd4998 100644 Binary files a/docs/images/18_setup_direct_proxy.png and b/docs/images/18_setup_direct_proxy.png differ diff --git a/docs/images/20_setup_webuser.png b/docs/images/20_setup_webuser.png index 893f253c..83878720 100644 Binary files a/docs/images/20_setup_webuser.png and b/docs/images/20_setup_webuser.png differ diff --git a/docs/images/21_setup_webpass1.png b/docs/images/21_setup_webpass1.png index bead11a5..e0614ed9 100644 Binary files a/docs/images/21_setup_webpass1.png and b/docs/images/21_setup_webpass1.png differ diff --git a/docs/images/22_setup_webpass2.png b/docs/images/22_setup_webpass2.png index ac366372..c227eab0 100644 Binary files a/docs/images/22_setup_webpass2.png and b/docs/images/22_setup_webpass2.png differ diff --git a/docs/images/23_setup_access_type.png b/docs/images/23_setup_access_type.png index 042ac244..9c81796b 100644 Binary files a/docs/images/23_setup_access_type.png and b/docs/images/23_setup_access_type.png differ diff --git a/docs/images/26_setup_so_allow.png b/docs/images/26_setup_so_allow.png index 7f95a73f..0d9fd451 100644 Binary files a/docs/images/26_setup_so_allow.png and b/docs/images/26_setup_so_allow.png differ diff --git a/docs/images/27_setup_so_allow_input.png b/docs/images/27_setup_so_allow_input.png index 1583a513..38381a3e 100644 Binary files a/docs/images/27_setup_so_allow_input.png and b/docs/images/27_setup_so_allow_input.png differ diff --git a/docs/images/27_telemetry.png b/docs/images/27_telemetry.png index 1af2a27a..9a02d1ec 100644 Binary files a/docs/images/27_telemetry.png and b/docs/images/27_telemetry.png differ diff --git a/docs/images/28_setup_summary.png b/docs/images/28_setup_summary.png index 47d93765..4db83921 100644 Binary files a/docs/images/28_setup_summary.png and b/docs/images/28_setup_summary.png differ diff --git a/docs/images/29_setup_finished.png b/docs/images/29_setup_finished.png index e0f5e458..9181678d 100644 Binary files a/docs/images/29_setup_finished.png and b/docs/images/29_setup_finished.png differ diff --git a/docs/images/37_login.png b/docs/images/37_login.png index 8df54328..999b2007 100644 Binary files a/docs/images/37_login.png and b/docs/images/37_login.png differ diff --git a/docs/images/38_overview.png b/docs/images/38_overview.png index f7dbd770..cef73b8c 100644 Binary files a/docs/images/38_overview.png and b/docs/images/38_overview.png differ diff --git a/docs/images/39_grid.png b/docs/images/39_grid.png index 4c738f00..82b46b79 100644 Binary files a/docs/images/39_grid.png and b/docs/images/39_grid.png differ diff --git a/docs/images/40_upload.png b/docs/images/40_upload.png index fd0335db..ffa6dd47 100644 Binary files a/docs/images/40_upload.png and b/docs/images/40_upload.png differ diff --git a/docs/images/45_import.png b/docs/images/45_import.png index 058b0e5b..dacb0415 100644 Binary files a/docs/images/45_import.png and b/docs/images/45_import.png differ diff --git a/docs/images/50_alerts.png b/docs/images/50_alerts.png index 9bedaf06..3af3be29 100644 Binary files a/docs/images/50_alerts.png and b/docs/images/50_alerts.png differ diff --git a/docs/images/51_alerts_play.png b/docs/images/51_alerts_play.png index 60fb4ec8..86baf29a 100644 Binary files a/docs/images/51_alerts_play.png and b/docs/images/51_alerts_play.png differ diff --git a/docs/images/52_alerts_options.png b/docs/images/52_alerts_options.png index 25da02a5..a5d23571 100644 Binary files a/docs/images/52_alerts_options.png and b/docs/images/52_alerts_options.png differ diff --git a/docs/images/53_dashboards.png b/docs/images/53_dashboards.png index 3536daef..25c36aca 100644 Binary files a/docs/images/53_dashboards.png and b/docs/images/53_dashboards.png differ diff --git a/docs/images/54_dashboards_options.png b/docs/images/54_dashboards_options.png index f78649f9..58b73276 100644 Binary files a/docs/images/54_dashboards_options.png and b/docs/images/54_dashboards_options.png differ diff --git a/docs/images/56_hunt.png b/docs/images/56_hunt.png index 9385a320..91f6ce71 100644 Binary files a/docs/images/56_hunt.png and b/docs/images/56_hunt.png differ diff --git a/docs/images/57_0_cases.png b/docs/images/57_0_cases.png index e9c5f209..ad326c52 100644 Binary files a/docs/images/57_0_cases.png and b/docs/images/57_0_cases.png differ diff --git a/docs/images/57_1_cases_options.png b/docs/images/57_1_cases_options.png index 65ed71a8..704d7f8a 100644 Binary files a/docs/images/57_1_cases_options.png and b/docs/images/57_1_cases_options.png differ diff --git a/docs/images/57_2_cases_create.png b/docs/images/57_2_cases_create.png index 7fc627ab..5f175ef7 100644 Binary files a/docs/images/57_2_cases_create.png and b/docs/images/57_2_cases_create.png differ diff --git a/docs/images/57_detections.png b/docs/images/57_detections.png index 9ffa05ad..bbd73eb4 100644 Binary files a/docs/images/57_detections.png and b/docs/images/57_detections.png differ diff --git a/docs/images/58_detections_options.png b/docs/images/58_detections_options.png index c3fa9b44..c94b5f25 100644 Binary files a/docs/images/58_detections_options.png and b/docs/images/58_detections_options.png differ diff --git a/docs/images/59_detection_create.png b/docs/images/59_detection_create.png index e54aa4d1..c694d55e 100644 Binary files a/docs/images/59_detection_create.png and b/docs/images/59_detection_create.png differ diff --git a/docs/images/60_detection_nids.png b/docs/images/60_detection_nids.png index 5b68442e..13cc3a7b 100644 Binary files a/docs/images/60_detection_nids.png and b/docs/images/60_detection_nids.png differ diff --git a/docs/images/60_detection_nids_0_comments.png b/docs/images/60_detection_nids_0_comments.png index a659d75c..097d1cce 100644 Binary files a/docs/images/60_detection_nids_0_comments.png and b/docs/images/60_detection_nids_0_comments.png differ diff --git a/docs/images/60_detection_nids_1_signature.png b/docs/images/60_detection_nids_1_signature.png index 46addfd9..d4213a31 100644 Binary files a/docs/images/60_detection_nids_1_signature.png and b/docs/images/60_detection_nids_1_signature.png differ diff --git a/docs/images/60_detection_nids_2_tuning_1.png b/docs/images/60_detection_nids_2_tuning_1.png index 4f9130ef..19cb3895 100644 Binary files a/docs/images/60_detection_nids_2_tuning_1.png and b/docs/images/60_detection_nids_2_tuning_1.png differ diff --git a/docs/images/60_detection_nids_2_tuning_2_add.png b/docs/images/60_detection_nids_2_tuning_2_add.png index 7b19e539..71399076 100644 Binary files a/docs/images/60_detection_nids_2_tuning_2_add.png and b/docs/images/60_detection_nids_2_tuning_2_add.png differ diff --git a/docs/images/60_detection_nids_3_playbook.png b/docs/images/60_detection_nids_3_playbook.png index e0d4329f..591191b8 100644 Binary files a/docs/images/60_detection_nids_3_playbook.png and b/docs/images/60_detection_nids_3_playbook.png differ diff --git a/docs/images/60_detection_nids_4_history.png b/docs/images/60_detection_nids_4_history.png index 0d3bcfb1..0d4a08e3 100644 Binary files a/docs/images/60_detection_nids_4_history.png and b/docs/images/60_detection_nids_4_history.png differ diff --git a/docs/images/60_detection_sigma.png b/docs/images/60_detection_sigma.png index 22b0eacb..06b739b6 100644 Binary files a/docs/images/60_detection_sigma.png and b/docs/images/60_detection_sigma.png differ diff --git a/docs/images/60_detection_sigma_2_tuning_1.png b/docs/images/60_detection_sigma_2_tuning_1.png index ca092b21..8e984836 100644 Binary files a/docs/images/60_detection_sigma_2_tuning_1.png and b/docs/images/60_detection_sigma_2_tuning_1.png differ diff --git a/docs/images/60_detection_sigma_2_tuning_2_add.png b/docs/images/60_detection_sigma_2_tuning_2_add.png index e3ff6f7a..ec4cc14d 100644 Binary files a/docs/images/60_detection_sigma_2_tuning_2_add.png and b/docs/images/60_detection_sigma_2_tuning_2_add.png differ diff --git a/docs/images/60_detection_yara.png b/docs/images/60_detection_yara.png index 00ed4d35..dc258ce9 100644 Binary files a/docs/images/60_detection_yara.png and b/docs/images/60_detection_yara.png differ diff --git a/docs/images/61_actions.png b/docs/images/61_actions.png index ac024569..7d97d3b6 100644 Binary files a/docs/images/61_actions.png and b/docs/images/61_actions.png differ diff --git a/docs/images/62_pcap.png b/docs/images/62_pcap.png index 7d68e43a..9e1c2f8f 100644 Binary files a/docs/images/62_pcap.png and b/docs/images/62_pcap.png differ diff --git a/docs/images/65_pcap_details.png b/docs/images/65_pcap_details.png index eaba3d0c..7f19876d 100644 Binary files a/docs/images/65_pcap_details.png and b/docs/images/65_pcap_details.png differ diff --git a/docs/images/72_jobs.png b/docs/images/72_jobs.png index 5d61c32d..a837a740 100644 Binary files a/docs/images/72_jobs.png and b/docs/images/72_jobs.png differ diff --git a/docs/images/73_jobs_add.png b/docs/images/73_jobs_add.png index 5daa32e4..e884b2f2 100644 Binary files a/docs/images/73_jobs_add.png and b/docs/images/73_jobs_add.png differ diff --git a/docs/images/75_grid.png b/docs/images/75_grid.png index 4c738f00..52ee88aa 100644 Binary files a/docs/images/75_grid.png and b/docs/images/75_grid.png differ diff --git a/docs/images/76_grid_options.png b/docs/images/76_grid_options.png index 9581019a..2e755561 100644 Binary files a/docs/images/76_grid_options.png and b/docs/images/76_grid_options.png differ diff --git a/docs/images/78_downloads.png b/docs/images/78_downloads.png index 2ec96b51..5baccc36 100644 Binary files a/docs/images/78_downloads.png and b/docs/images/78_downloads.png differ diff --git a/docs/images/81_users.png b/docs/images/81_users.png index 9538d003..3630b8dc 100644 Binary files a/docs/images/81_users.png and b/docs/images/81_users.png differ diff --git a/docs/images/82_users_detail.png b/docs/images/82_users_detail.png index 22682bdd..2b77c423 100644 Binary files a/docs/images/82_users_detail.png and b/docs/images/82_users_detail.png differ diff --git a/docs/images/83_users_add.png b/docs/images/83_users_add.png index 28422d7b..fa8807b2 100644 Binary files a/docs/images/83_users_add.png and b/docs/images/83_users_add.png differ diff --git a/docs/images/84_gridmembers.png b/docs/images/84_gridmembers.png index 4557c232..8851faac 100644 Binary files a/docs/images/84_gridmembers.png and b/docs/images/84_gridmembers.png differ diff --git a/docs/images/87_config.png b/docs/images/87_config.png index 6ac8cbbf..deb7b490 100644 Binary files a/docs/images/87_config.png and b/docs/images/87_config.png differ diff --git a/docs/images/88_config_options.png b/docs/images/88_config_options.png index 73adfda7..89d7396f 100644 Binary files a/docs/images/88_config_options.png and b/docs/images/88_config_options.png differ diff --git a/docs/images/91_licensekey.png b/docs/images/91_licensekey.png index 9b074472..bbf1e247 100644 Binary files a/docs/images/91_licensekey.png and b/docs/images/91_licensekey.png differ diff --git a/docs/images/94_usermenu.png b/docs/images/94_usermenu.png index 34b4118a..a93ea630 100644 Binary files a/docs/images/94_usermenu.png and b/docs/images/94_usermenu.png differ diff --git a/docs/images/97_navigator.png b/docs/images/97_navigator.png index a460b83b..19f0d69c 100644 Binary files a/docs/images/97_navigator.png and b/docs/images/97_navigator.png differ diff --git a/docs/images/cheat-sheet/Security-Onion-Cheat-Sheet.pdf b/docs/images/cheat-sheet/Security-Onion-Cheat-Sheet.pdf index b8b1d4e1..b67cd554 100644 Binary files a/docs/images/cheat-sheet/Security-Onion-Cheat-Sheet.pdf and b/docs/images/cheat-sheet/Security-Onion-Cheat-Sheet.pdf differ diff --git a/docs/images/config-item-backup.png b/docs/images/config-item-backup.png index d1234e19..41a7f482 100644 Binary files a/docs/images/config-item-backup.png and b/docs/images/config-item-backup.png differ diff --git a/docs/images/config-item-bpf.png b/docs/images/config-item-bpf.png index 4f5da71e..19666a32 100644 Binary files a/docs/images/config-item-bpf.png and b/docs/images/config-item-bpf.png differ diff --git a/docs/images/config-item-elastalert-alerter.png b/docs/images/config-item-elastalert-alerter.png index 5c5c0e96..73288f79 100644 Binary files a/docs/images/config-item-elastalert-alerter.png and b/docs/images/config-item-elastalert-alerter.png differ diff --git a/docs/images/config-item-elastalert.png b/docs/images/config-item-elastalert.png index 0fd6928a..354d2175 100644 Binary files a/docs/images/config-item-elastalert.png and b/docs/images/config-item-elastalert.png differ diff --git a/docs/images/config-item-elasticfleet.png b/docs/images/config-item-elasticfleet.png index 7f40f98f..3160a09b 100644 Binary files a/docs/images/config-item-elasticfleet.png and b/docs/images/config-item-elasticfleet.png differ diff --git a/docs/images/config-item-elasticsearch.png b/docs/images/config-item-elasticsearch.png index 2640f93e..b7c8a71d 100644 Binary files a/docs/images/config-item-elasticsearch.png and b/docs/images/config-item-elasticsearch.png differ diff --git a/docs/images/config-item-firewall.png b/docs/images/config-item-firewall.png index f1563211..b1a89158 100644 Binary files a/docs/images/config-item-firewall.png and b/docs/images/config-item-firewall.png differ diff --git a/docs/images/config-item-global-url.png b/docs/images/config-item-global-url.png index 4c9b7a59..f4ae2c6f 100644 Binary files a/docs/images/config-item-global-url.png and b/docs/images/config-item-global-url.png differ diff --git a/docs/images/config-item-global.png b/docs/images/config-item-global.png index ae76a0fd..2114094f 100644 Binary files a/docs/images/config-item-global.png and b/docs/images/config-item-global.png differ diff --git a/docs/images/config-item-host.png b/docs/images/config-item-host.png index 62572618..9d4fe0af 100644 Binary files a/docs/images/config-item-host.png and b/docs/images/config-item-host.png differ diff --git a/docs/images/config-item-idh.png b/docs/images/config-item-idh.png index c3b597e3..4b56a4d5 100644 Binary files a/docs/images/config-item-idh.png and b/docs/images/config-item-idh.png differ diff --git a/docs/images/config-item-influxdb.png b/docs/images/config-item-influxdb.png index b1dedc0a..b29ea9b3 100644 Binary files a/docs/images/config-item-influxdb.png and b/docs/images/config-item-influxdb.png differ diff --git a/docs/images/config-item-kafka.png b/docs/images/config-item-kafka.png index 9ae6385e..b0e09366 100644 Binary files a/docs/images/config-item-kafka.png and b/docs/images/config-item-kafka.png differ diff --git a/docs/images/config-item-kibana.png b/docs/images/config-item-kibana.png index c54fb0fd..eebd99ea 100644 Binary files a/docs/images/config-item-kibana.png and b/docs/images/config-item-kibana.png differ diff --git a/docs/images/config-item-kratos.png b/docs/images/config-item-kratos.png index e687e500..a1bd9bd9 100644 Binary files a/docs/images/config-item-kratos.png and b/docs/images/config-item-kratos.png differ diff --git a/docs/images/config-item-logstash.png b/docs/images/config-item-logstash.png index 111131ee..b3770385 100644 Binary files a/docs/images/config-item-logstash.png and b/docs/images/config-item-logstash.png differ diff --git a/docs/images/config-item-manager.png b/docs/images/config-item-manager.png index a87a1969..0b4d3d0c 100644 Binary files a/docs/images/config-item-manager.png and b/docs/images/config-item-manager.png differ diff --git a/docs/images/config-item-nginx.png b/docs/images/config-item-nginx.png index 3c6313a7..d69b319f 100644 Binary files a/docs/images/config-item-nginx.png and b/docs/images/config-item-nginx.png differ diff --git a/docs/images/config-item-ntp.png b/docs/images/config-item-ntp.png index 19757a37..7f6a7017 100644 Binary files a/docs/images/config-item-ntp.png and b/docs/images/config-item-ntp.png differ diff --git a/docs/images/config-item-patch.png b/docs/images/config-item-patch.png index b3c175e0..cfd3866f 100644 Binary files a/docs/images/config-item-patch.png and b/docs/images/config-item-patch.png differ diff --git a/docs/images/config-item-redis.png b/docs/images/config-item-redis.png index 33376e55..bbf04408 100644 Binary files a/docs/images/config-item-redis.png and b/docs/images/config-item-redis.png differ diff --git a/docs/images/config-item-sensor.png b/docs/images/config-item-sensor.png index 3bc466a8..593474b7 100644 Binary files a/docs/images/config-item-sensor.png and b/docs/images/config-item-sensor.png differ diff --git a/docs/images/config-item-sensoroni.png b/docs/images/config-item-sensoroni.png index 4edbb3d1..a9a7c34d 100644 Binary files a/docs/images/config-item-sensoroni.png and b/docs/images/config-item-sensoroni.png differ diff --git a/docs/images/config-item-soc-additionalAlerters.png b/docs/images/config-item-soc-additionalAlerters.png index 1bcea7bb..da4390a8 100644 Binary files a/docs/images/config-item-soc-additionalAlerters.png and b/docs/images/config-item-soc-additionalAlerters.png differ diff --git a/docs/images/config-item-soc-subgrids.png b/docs/images/config-item-soc-subgrids.png index 9849adbf..3dec4024 100644 Binary files a/docs/images/config-item-soc-subgrids.png and b/docs/images/config-item-soc-subgrids.png differ diff --git a/docs/images/config-item-soc.png b/docs/images/config-item-soc.png index fb557022..76a828f5 100644 Binary files a/docs/images/config-item-soc.png and b/docs/images/config-item-soc.png differ diff --git a/docs/images/config-item-strelka.png b/docs/images/config-item-strelka.png index 34f8de71..e6afd1c7 100644 Binary files a/docs/images/config-item-strelka.png and b/docs/images/config-item-strelka.png differ diff --git a/docs/images/config-item-suricata.png b/docs/images/config-item-suricata.png index be8991e4..846222a7 100644 Binary files a/docs/images/config-item-suricata.png and b/docs/images/config-item-suricata.png differ diff --git a/docs/images/config-item-telegraf.png b/docs/images/config-item-telegraf.png index 1c7441e7..cb044b7c 100644 Binary files a/docs/images/config-item-telegraf.png and b/docs/images/config-item-telegraf.png differ diff --git a/docs/images/config-item-versionlock.png b/docs/images/config-item-versionlock.png index 228ca725..6a31ea60 100644 Binary files a/docs/images/config-item-versionlock.png and b/docs/images/config-item-versionlock.png differ diff --git a/docs/images/config-item-zeek.png b/docs/images/config-item-zeek.png index 40ea859d..18dc603e 100644 Binary files a/docs/images/config-item-zeek.png and b/docs/images/config-item-zeek.png differ diff --git a/docs/images/diagrams/analyst.png b/docs/images/diagrams/analyst.png index 2eb23105..9f0b7745 100644 Binary files a/docs/images/diagrams/analyst.png and b/docs/images/diagrams/analyst.png differ diff --git a/docs/images/diagrams/distributed.png b/docs/images/diagrams/distributed.png index 8a97cde0..238f8f1a 100644 Binary files a/docs/images/diagrams/distributed.png and b/docs/images/diagrams/distributed.png differ diff --git a/docs/images/diagrams/eval.png b/docs/images/diagrams/eval.png index 83465ab0..f97301eb 100644 Binary files a/docs/images/diagrams/eval.png and b/docs/images/diagrams/eval.png differ diff --git a/docs/images/diagrams/heavy-distributed.png b/docs/images/diagrams/heavy-distributed.png index 885a12cc..e66dd1e0 100644 Binary files a/docs/images/diagrams/heavy-distributed.png and b/docs/images/diagrams/heavy-distributed.png differ diff --git a/docs/images/diagrams/idh.png b/docs/images/diagrams/idh.png index cafb2fdf..1664566e 100644 Binary files a/docs/images/diagrams/idh.png and b/docs/images/diagrams/idh.png differ diff --git a/docs/images/diagrams/import.png b/docs/images/diagrams/import.png index 136b999a..f3a61f01 100644 Binary files a/docs/images/diagrams/import.png and b/docs/images/diagrams/import.png differ diff --git a/docs/images/diagrams/network-horiz.png b/docs/images/diagrams/network-horiz.png index 25d89355..9fd06d5b 100644 Binary files a/docs/images/diagrams/network-horiz.png and b/docs/images/diagrams/network-horiz.png differ diff --git a/docs/images/diagrams/receiver.png b/docs/images/diagrams/receiver.png index 27d47312..ab60390b 100644 Binary files a/docs/images/diagrams/receiver.png and b/docs/images/diagrams/receiver.png differ diff --git a/docs/images/diagrams/sniffing.png b/docs/images/diagrams/sniffing.png index 52281fb9..3e7e153f 100644 Binary files a/docs/images/diagrams/sniffing.png and b/docs/images/diagrams/sniffing.png differ diff --git a/docs/images/diagrams/standalone.png b/docs/images/diagrams/standalone.png index c5699447..6734ad64 100644 Binary files a/docs/images/diagrams/standalone.png and b/docs/images/diagrams/standalone.png differ diff --git a/docs/images/logo/favicon.ico b/docs/images/logo/favicon.ico new file mode 100644 index 00000000..8bc70f5d Binary files /dev/null and b/docs/images/logo/favicon.ico differ diff --git a/docs/index.md b/docs/index.md index ca07156f..83befda9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,34 +1,56 @@ -# Security Onion Documentation - -Welcome to Security Onion! - -This is the MAIN branch. - -## Table of Contents - -- [About](about.md) -- [Introduction](introduction.md) -- [License](license.md) -- [First Time Users](first-time-users.md) -- [Getting Started](getting-started.md) -- [Security Onion Console](security-onion-console.md) -- [Security Onion Desktop](security-onion-desktop.md) -- [Network](network-visibility.md) -- [Additional Network](additional-network-visibility.md) -- [Host](host-visibility.md) -- [Third Party Integrations](third-party-integrations.md) -- [Rules](rules.md) -- [Logs](logs.md) -- [Updating](updating.md) -- [Accounts](accounts.md) -- [Services](services.md) -- [Customizing](customizing.md) -- [Tricks and Tips](tricks-and-tips.md) -- [Utilities](utilities.md) -- [Help](help.md) -- [Security Onion Pro](security-onion-pro.md) -- [Security](security.md) -- [Telemetry](telemetry.md) -- [Release Notes](release-notes.md) -- [Appendix](appendix.md) -- [Cheat Sheet](cheat-sheet.md) +# About + +## Security Onion + +Security Onion is a free and open platform built by defenders for defenders. It includes [network visibility](network-visibility.md), [host visibility](host-visibility.md), [intrusion detection honeypots](idh.md), [log management](elasticsearch.md), and [case management](cases.md). Security Onion has been downloaded over 2 million times and is being used by security teams around the world to monitor and defend their enterprises. Our easy-to-use Setup wizard allows you to build a distributed grid for your enterprise in minutes! + +## Security Onion Solutions, LLC + +Doug Burks started Security Onion as a free and open project in 2008 and then founded Security Onion Solutions, LLC in 2014. + +!!! IMPORTANT + + Security Onion Solutions, LLC is the only official provider of hardware appliances, training, and professional services for Security Onion. + +For more information about these products and services, please see our company site at . + +## Documentation + +!!! WARNING + + Documentation is always a work in progress and some documentation may be missing or incorrect. Please let us know if you notice any issues. + +### License + +This documentation is licensed under CC BY 4.0. You can read more about this license at . + +### Formats + +This documentation is published online at . If you are viewing an offline version of this documentation but have Internet access, you might want to switch to the online version at to see the latest version. + +This documentation is also available in PDF format at . + +Many folks have asked for a printed version of our documentation. Whether you work on airgapped networks or simply want a portable reference that doesn't require an Internet connection or batteries, this is what you've been asking for. Thanks to Richard Bejtlich for writing the inspiring foreword! Proceeds go to the Rural Technology Fund! You can purchase your copy at . + +### Authors + +Security Onion Solutions is the primary author and maintainer of this documentation. Some content has been contributed by members of our community. Thanks to all the folks who have contributed to this documentation over the years! + +### Contributing + +We welcome your contributions to our documentation! We will review any suggestions and apply them if appropriate. + +If you are accessing the online version of the documentation and notice that a particular page has incorrect information, you can submit corrections by clicking the `Edit on GitHub` button in the upper-right corner of each page. Once you have made your corrections, you will need to submit your pull request (PR) to the `dev` branch. + +To submit a new page, you can submit a pull request (PR) to the `3/dev` branch of the `docs` repo at . + +Pages are written in Markdown format and you can find several Markdown guides on the Internet including . + +### Naming Convention + +New documentation pages should use the following naming convention: + +- all lowercase +- `.md` file extension +- ideally, the name of the page should be one simple word (for example: `suricata.md`) +- if necessary, the name of the page can be hyphenated (for example: network-visibility.md) diff --git a/docs/influxdb.md b/docs/influxdb.md index 05620800..9cd82f80 100644 --- a/docs/influxdb.md +++ b/docs/influxdb.md @@ -1,6 +1,6 @@ # InfluxDB -[SOC](security-onion-console.md) includes a link on the sidebar that takes you to InfluxDB. +[Security Onion Console](security-onion-console.md) includes a link on the sidebar that takes you to InfluxDB. From : @@ -26,4 +26,4 @@ You can configure Telegraf by going to [Administration](administration.md) --> C !!! NOTE - For more information about InfluxDB, please see . \ No newline at end of file + For more information about InfluxDB, please see . diff --git a/docs/ingest.md b/docs/ingest.md index 6c089e56..5c7ca074 100644 --- a/docs/ingest.md +++ b/docs/ingest.md @@ -5,56 +5,69 @@ Here's an overview of how logs are ingested in various deployment types. ## Import **Core Pipeline:** Elastic Agent [IMPORT Node] --> Elasticsearch Ingest [IMPORT Node] + **Logs:** Zeek, Suricata ## Eval **Core Pipeline:** Elastic Agent [EVAL Node] --> Elasticsearch Ingest [EVAL Node] + **Logs:** Zeek, Suricata ## Standalone **Core Pipeline:** Elastic Agent [SA Node] --> Logstash [SA Node] --> Redis [SA Node] <--> Logstash [SA Node] --> Elasticsearch Ingest [SA Node] + **Logs:** Zeek, Suricata, syslog **Elastic Agent:** Elastic Agent [Windows Endpoint] --> Logstash [SA Node] --> Redis [SA Node] <--> Logstash [SA Node] --> Elasticsearch Ingest [SA Node] + **Logs:** WEL, Sysmon ## Fleet Standalone **Pipeline:** Elastic Agent [Fleet Node] --> Logstash [M | MS] --> Elasticsearch Ingest [S | MS] + **Logs:** Elastic Agent ## Manager (separate search nodes) **Core Pipeline:** Elastic Agent [Fleet | Sensor] --> Logstash [Manager] --> Redis [Manager] + **Logs:** Zeek, Suricata, syslog **Elastic Agent:** Elastic Agent [Windows Endpoint] --> Logstash [Manager] --> Redis [Manager] + **Logs:** WEL, Sysmon ## Manager Search **Core Pipeline:** Elastic Agent [Fleet | Sensor] --> Logstash [MS] --> Redis [MS] <--> Logstash [MS] --> Elasticsearch Ingest [MS] + **Logs:** Zeek, Suricata, syslog **Pipeline:** Elastic Agent [MS] --> Logstash [MS] --> Elasticsearch Ingest [MS] + **Logs:** Local Elastic Agent **Elastic Agent:** Elastic Agent [Windows Endpoint] --> Logstash [MS] --> Elasticsearch Ingest [MS] + **Logs:** WEL, Sysmon ## Heavy **Pipeline:** Elastic Agent [Heavy Node] --> Elasticsearch Ingest [Heavy] + **Logs:** Zeek, Suricata, syslog ## Search **Pipeline:** Redis [Manager] --> Logstash [Search] --> Elasticsearch Ingest [Search] + **Logs:** Zeek, Suricata, syslog ## Sensor **Pipeline:** Elastic Agent [Sensor] --> Logstash [M | MS] --> Elasticsearch Ingest [S | MS] -**Logs:** Zeek, Suricata, syslog \ No newline at end of file + +**Logs:** Zeek, Suricata, syslog diff --git a/docs/introduction.md b/docs/introduction.md index bc2ccadf..70a9fad5 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -2,7 +2,7 @@ Security Onion is a free and open platform built by defenders for defenders. It includes [network visibility](network-visibility.md), [host visibility](host-visibility.md), [intrusion detection honeypots](idh.md), [log management](elasticsearch.md), and [case management](cases.md). -For network visibility, we offer signature based detection via [Suricata](suricata.md), rich protocol metadata and file extraction using either [Zeek](zeek.md) or [Suricata](suricata.md), full packet capture using [Suricata](suricata.md), and file analysis. For host visibility, we offer the [Elastic Agent](elastic-agent.md) which provides data collection, live queries via [Osquery](osquery-manager.md), and centralized management using [Elastic Fleet](elastic-fleet.md). [Intrusion detection honeypots](idh.md) based on OpenCanary can be added to your deployment for even more enterprise visibility. All of these logs flow into [Elasticsearch](elasticsearch.md) and we've built our own user interfaces for [Alerts](alerts.md), [Dashboards](dashboards.md), [threat hunting](hunt.md), [case management](cases.md), and [Grid management](grid.md). +For network visibility, we offer signature based detection via [Suricata](suricata.md), rich protocol metadata and file extraction using either [Zeek](zeek.md) or [Suricata](suricata.md), full packet capture using [Suricata](suricata.md), and file analysis. For host visibility, we offer the [Elastic Agent](elastic-agent.md) which provides data collection, live queries via [Osquery](osquery-manager.md), and centralized management using [Elastic Fleet](elastic-fleet.md). [Intrusion detection honeypots](idh.md) based on OpenCanary can be added to your deployment for even more enterprise visibility. All of these logs flow into [Elasticsearch](elasticsearch.md) and we've built our own user interfaces for [Alerts](alerts.md), [Dashboards](dashboards.md), [threat hunting](hunt.md), [case management](cases.md), and [grid management](grid.md). !!! NOTE @@ -97,4 +97,4 @@ Analysts around the world are using Security Onion today for many different [arc ## Conclusion -After you install Security Onion, you will have comprehensive network and host visibility for your enterprise. Our analyst tools will enable you to use all of that data to detect intruders more quickly and paint a more complete picture of what they're doing in your environment. Get ready to peel back the layers of your enterprise and make your adversaries cry! \ No newline at end of file +After you install Security Onion, you will have comprehensive network and host visibility for your enterprise. Our analyst tools will enable you to use all of that data to detect intruders more quickly and paint a more complete picture of what they're doing in your environment. Get ready to peel back the layers of your enterprise and make your adversaries cry! diff --git a/docs/ip.md b/docs/ip-address.md similarity index 100% rename from docs/ip.md rename to docs/ip-address.md diff --git a/docs/iptables.md b/docs/iptables.md index 050f93fe..98f7dbac 100644 --- a/docs/iptables.md +++ b/docs/iptables.md @@ -10,7 +10,7 @@ First, add the Elastic integration for `iptables`. For more information about the `iptables` integration, please see . -1. Go to [Elastic Fleet](elastic-fleet.md), click the `Agent policies` tab, and then click the desired policy (for example `so-Grid-nodes_general`). +1. Go to [Elastic Fleet](elastic-fleet.md), click the `Agent policies` tab, and then click the desired policy (for example `so-grid-nodes_general`). 2. Click the `Add integration` button. 3. Search for `iptables` and then click on the `iptables` integration. 4. The Elastic Integration page will show an overview of the iptables Integration. Review all information on the page and then click the `Add iptables` button. @@ -29,7 +29,7 @@ Next, allow the traffic from the iptables host through the firewall to the iptab 3. On the left side, go to `firewall`, select `hostgroups`, and click the `customhostgroup0` group. On the right side, enter the IP address of the iptables host and click the checkmark to save. 4. On the left side, go to `firewall`, select `portgroups`, select the `customportgroup0` group, and then click `udp`. On the right side, enter your desired listener port (9001 by default) and click the checkmark to save. 5. On the left side, go to `firewall`, select `role`, and then select the node type that will receive the iptables logs. Then drill into `chain` --> `INPUT` --> `hostgroups` --> `customhostgroup0` --> `portgroups`. On the right side, enter `customportgroup0` and click the checkmark to save. -6. If you would like to apply the rules immediately, click the `SYNCHRONIZE Grid` button under the `Options` menu at the top of the page. +6. If you would like to apply the rules immediately, click the `SYNCHRONIZE GRID` button under the `Options` menu at the top of the page. ## iptables dashboard diff --git a/docs/kafka.md b/docs/kafka.md index 71c77b13..31bcf815 100644 --- a/docs/kafka.md +++ b/docs/kafka.md @@ -26,7 +26,7 @@ For more information about replication, please see Configuration --> global --> pipeline and setting the value to `KAFKA`. -There is no need to click on the `SYNCHRONIZE Grid` button. Once you have set the global pipeline value to `KAFKA`, the changes will begin to take effect in the background before finally switching the Grid to the new pipeline. +There is no need to click on the `SYNCHRONIZE GRID` button. Once you have set the global pipeline value to `KAFKA`, the changes will begin to take effect in the background before finally switching the grid to the new pipeline. !!! NOTE diff --git a/docs/kernels.md b/docs/kernels.md new file mode 100644 index 00000000..b2b80d7d --- /dev/null +++ b/docs/kernels.md @@ -0,0 +1,39 @@ +# Kernels + +Security Onion standardizes on the Oracle Unbreakable Enterprise Kernel (UEK). Nodes run the UEK8 (6.x) kernel series, and the standard EL9 RedHat compatible kernel (RHCK, 5.14) is removed once a node is actually running UEK8. + +This is all automatic and you should not need to do anything. Nodes pick up the UEK8 kernel as part of their normal OS updates, and the stock EL9 kernel is removed for you on the next highstate after the node reboots onto UEK8. + +The removal is deliberately deferred until after the reboot. `dnf` refuses to erase the running kernel, and waiting also means the node has proven it boots on UEK8 before its fallback is deleted. Fresh installs reboot at the end of setup and are cleaned up on the first highstate after that; existing nodes are cleaned up whenever you reboot them. + +These are the packages removed: + +``` +kernel kernel-core kernel-modules kernel-modules-core kernel-tools kernel-tools-libs +``` + +!!! NOTE + + Older UEK7 `kernel-uek` 5.x packages are not removed. They age out on their own as newer kernels are installed. + +## Doing It Manually + +If a node did not end up on UEK8 on its own, you can run the following command on that node: + +``` +sudo so-kernel-upgrade +``` + +This installs the UEK8 kernel and makes it the boot default. It does not reboot the node, so you can schedule the reboot yourself. The new kernel does not take effect until you reboot. + +!!! NOTE + + The manager mirrors the UEK8 packages and serves them to the rest of the grid. If the manager has not synced them yet, `so-kernel-upgrade` on the manager will sync them for you; on any other node it will tell you to sync the manager first. + +Similarly, if the stock EL9 kernel is still installed on a node that is already running UEK8, you can run the cleanup directly instead of waiting for the next highstate: + +``` +sudo so-kernel-upgrade --cleanup +``` + +This does nothing unless the node is already running UEK8. diff --git a/docs/kibana.md b/docs/kibana.md index 8b5074fd..c9979edd 100644 --- a/docs/kibana.md +++ b/docs/kibana.md @@ -1,6 +1,6 @@ # Kibana -[SOC](security-onion-console.md) includes a link on the sidebar that takes you to Kibana. +[Security Onion Console](security-onion-console.md) includes a link on the sidebar that takes you to Kibana. ## Authentication @@ -76,4 +76,4 @@ You can enable or disable specific features by clicking the main menu in the upp !!! NOTE - For more information about Kibana, please see [https://www.elastic.co/kibana](https://www.elastic.co/kibana). \ No newline at end of file + For more information about Kibana, please see [https://www.elastic.co/kibana](https://www.elastic.co/kibana). diff --git a/docs/local-llm.md b/docs/local-llm.md new file mode 100644 index 00000000..6b2b3d68 --- /dev/null +++ b/docs/local-llm.md @@ -0,0 +1,334 @@ +# Local LLM Hosting + +!!! NOTE + + [Onion AI](onion-ai.md) is an enterprise-level feature of Security Onion. Contact Security Onion Solutions, LLC via our website at for more information about purchasing a [Security Onion Pro](security-onion-pro.md) license. + +[Onion AI](onion-ai.md) can connect to a model you host yourself through the *OpenAI Chat* adapter. This page walks through building such an endpoint on a single AMD Strix Halo machine using `llama.cpp`, which is packaged and GPU-accelerated on that platform. + +The result is an OpenAI-compatible API on your own network, protected by an API key, serving a model with a context window large enough for the assistant to work with. + +!!! NOTE + + Hosting your own model is optional. A local model will not match the accuracy of the proprietary foundational models available through the SOAI adapter. See [Local Model Considerations](onion-ai.md#local-model-considerations) before deciding. + +## Hardware + +The reference platform for this guide is an AMD Strix Halo system (Ryzen AI Max series) with 128GB of unified memory. + +Strix Halo is a good fit for this task because its memory is shared between the CPU and the integrated GPU. Rather than being limited to the memory on a discrete graphics card, the GPU can address the bulk of system RAM, which is what makes it possible to hold a large model and a large context window at once on a single, relatively inexpensive machine. + +This guide assumes the AMD Ryzen AI Developer Platform operating system, which is Debian-based and ships with the ROCm stack and GPU-enabled `llama.cpp` packages already available. + +## Verify GPU Access + +Before installing anything, confirm the GPU is visible to the ROCm backend: + +```bash +llama-cli --list-devices +``` + +You should see a ROCm device with most of your system memory available to it: + +```text +Available devices: + ROCm0: Radeon 8060S Graphics (96454 MiB, 124565 MiB free) +``` + +If no ROCm device is listed, the GPU backend is not working and nothing later in this guide will run on the GPU. + +### Available GPU Memory + +The amount of memory the GPU may address is the GTT (Graphics Translation Table) size: + +```bash +cat /sys/class/drm/card0/device/mem_info_gtt_total +``` + +On the reference platform this reports roughly 101GB out of 128GB of system memory, which is ample and requires no tuning. If your system reports a much smaller value, you can raise it with the `amdgpu.gttsize` and `ttm.pages_limit` kernel parameters, or by increasing the memory allocated to graphics in your system firmware. + +## Install llama.cpp + +```bash +sudo apt install llama.cpp +``` + +This is a small metapackage that pulls in `llama.cpp-tools`, which provides `llama-server` and `llama-cli`. The HIP backend that provides GPU acceleration comes from the same vendor repository, so it stays current through normal system updates. + +The package also installs a `llama-server` systemd service, which is what you will configure below rather than running the server by hand. + +## Choose a Model + +This guide uses **Gemma 4 26B A4B**, a mixture-of-experts model with 26 billion total parameters but only about 4 billion active per token. That combination suits this hardware well: the full model must fit in memory, but the amount of data read per generated token stays small, which is what governs speed on a memory-bandwidth-limited system. + +It also supports a 262,144 token context window, comfortably above the 128k minimum recommended for the assistant. + +Pre-converted GGUF files are published by Unsloth. Pick a quantization based on how you want to trade accuracy against speed and memory: + +| Quantization | Size | Notes | +|---|---|---| +| `unsloth/gemma-4-26B-A4B-it-qat-GGUF:UD-Q4_K_XL` | 14.2 GB | Quantization-aware trained by Google. Best quality available at 4-bit, and the fastest option. | +| `unsloth/gemma-4-26B-A4B-it-GGUF:UD-Q4_K_M` | 16.9 GB | Standard 4-bit. | +| `unsloth/gemma-4-26B-A4B-it-GGUF:Q8_0` | 26.9 GB | Effectively indistinguishable from full precision. The recommended default when you have the memory. | +| `unsloth/gemma-4-26B-A4B-it-GGUF:BF16` | 50.5 GB | Full precision. Fits, but offers no meaningful accuracy gain over `Q8_0` while running slower. | + +!!! TIP + + Start with `Q8_0`. On a 128GB machine it leaves plenty of room for a full-size context window, and accuracy matters more than raw speed for security analysis. Move to the QAT 4-bit build if you need faster responses or want to leave more memory free. + +## Generate an API Key + +Never expose an inference endpoint without authentication. Generate a random key and store it in a file that only the service can read: + +```bash +sudo install -d -m 0755 /etc/llama-server +openssl rand -hex 32 | sudo tee /etc/llama-server/api-key > /dev/null +sudo chown root:_llama-server /etc/llama-server/api-key +sudo chmod 0640 /etc/llama-server/api-key +``` + +Display the key when you need it for the Onion AI adapter configuration: + +```bash +sudo cat /etc/llama-server/api-key +``` + +## Grant the Service Access to the GPU + +The packaged service runs as the unprivileged `_llama-server` user. Interactive login sessions are granted access to the GPU devices automatically through an ACL, but system services are not, so the service account must be added to the `render` and `video` groups: + +```bash +sudo usermod -aG render,video _llama-server +``` + +!!! WARNING + + This step is easy to miss. Without it the service starts and appears healthy, but silently falls back to CPU inference. The giveaway is `failed to initialize ROCm: no ROCm-capable device is detected` in the service log, followed by `no usable GPU found, --gpu-layers option will be ignored`. On this hardware, CPU-only inference is too slow to be usable. + +## Configure the Server + +The service reads its settings from `/etc/default/llama-server` as `LLAMA_ARG_*` environment variables, each corresponding to a `llama-server` command line option. Replace the contents of that file with: + +```bash +LLAMA_ARG_HOST=0.0.0.0 +LLAMA_ARG_PORT=8080 +LLAMA_ARG_HF_REPO=unsloth/gemma-4-26B-A4B-it-GGUF:Q8_0 +LLAMA_ARG_ALIAS=gemma-4-26b +LLAMA_ARG_CTX_SIZE=262144 +LLAMA_ARG_N_GPU_LAYERS=99 +LLAMA_ARG_FLASH_ATTN=on +LLAMA_ARG_N_PARALLEL=1 +LLAMA_ARG_API_KEY_FILE=/etc/llama-server/api-key +``` + +What these do: + +- **`LLAMA_ARG_HOST`** -- binds to all interfaces so the Security Onion manager can reach the endpoint. See [Network Access](#network-access) below. +- **`LLAMA_ARG_HF_REPO`** -- the model to serve. It is downloaded automatically on first start into `/var/cache/llama-server`. +- **`LLAMA_ARG_ALIAS`** -- the name the API reports for the model. This is the value you will enter as the model identifier in Onion AI. +- **`LLAMA_ARG_CTX_SIZE`** -- the context window, in tokens. `262144` is the maximum this model supports. +- **`LLAMA_ARG_N_GPU_LAYERS`** -- any value at or above the model's layer count offloads the whole model to the GPU. +- **`LLAMA_ARG_N_PARALLEL`** -- how many requests are served concurrently. The context window is divided among these slots, so leave it at `1` to give a single conversation the entire window. + +!!! NOTE + + Do not set `LLAMA_ARG_SWA_FULL`. Most of this model's layers use sliding-window attention, and leaving that option off allows the server to allocate a much smaller cache for them. Enabling it would make a 262,144 token context far more expensive in memory for no benefit. + +Tool calling, which the assistant depends on, requires the Jinja chat template engine. It is enabled by default, so no setting is needed, but do not disable it. + +## Start the Server + +```bash +sudo systemctl enable --now llama-server +``` + +The first start downloads the model, which takes a while. Watch its progress: + +```bash +journalctl -u llama-server -f +``` + +Once the model has loaded, confirm the server is answering: + +```bash +curl -s -H "Authorization: Bearer $(sudo cat /etc/llama-server/api-key)" \ + http://127.0.0.1:8080/v1/models +``` + +## Network Access + +The Security Onion manager connects to this endpoint over your network, so the port must be reachable from it. + +!!! WARNING + + The configuration above serves plain HTTP. The API key is sent as a bearer token in clear text and is readable by anyone who can observe the traffic, as are the prompts and responses, which will contain data from your grid. + + Restrict access to the manager and terminate TLS in front of the server before using this outside an isolated lab network. + +Limit which hosts may reach the port. If the machine has no firewall configured, allowing only the manager is a reasonable starting point: + +```bash +sudo nft add table inet filter +sudo nft add chain inet filter input '{ type filter hook input priority 0; }' +sudo nft add rule inet filter input tcp dport 8080 ip saddr != drop +``` + +For anything beyond a lab, put a reverse proxy such as nginx in front of `llama-server` to terminate TLS, and bind `llama-server` itself to `127.0.0.1` by setting `LLAMA_ARG_HOST=127.0.0.1`. + +## Connect to Onion AI + +With the endpoint running, configure Security Onion to use it. This mirrors the general instructions in [Onion AI](onion-ai.md#configuration). + +### Add the Adapter + +Go to Administration --> Configuration, turn on *Show advanced settings* in the Options at the top of the page, and navigate to soc --> config --> server --> modules --> assistant --> adapters. Add an adapter: + +| Field | Value | +|---|---| +| Adapter Name | `local` | +| Protocol | *OpenAI Chat* | +| API Url | `http://:8080/v1/` | +| API Key | the contents of `/etc/llama-server/api-key` | +| Health Timeout Seconds | `120` | + +!!! NOTE + + Use the *OpenAI Chat* adapter, not *OpenAI Responses*. `llama-server` implements the Chat Completions protocol. + + A generous health timeout is appropriate here. A locally hosted model on this class of hardware responds more slowly than a cloud provider, particularly when processing a large context. + +### Add the Model + +Navigate to soc --> config --> server --> client --> assistant --> availableModels and add an entry: + +| Field | Value | +|---|---| +| Name | `Gemma 4 26B (local)` | +| Adapter | `local` | +| Model | `gemma-4-26b` | + +The model identifier must match the `LLAMA_ARG_ALIAS` value you configured, since that is the name the server reports over the API. + +Finally, make sure the assistant itself is enabled at soc --> config --> server --> client --> assistant --> enabled, then open the assistant in SOC and select your new model. + +## Reasoning Output + +Gemma 4 is a reasoning model. It works through a problem before committing to an answer, and `llama-server` returns that internal reasoning in a separate `reasoning_content` field, leaving `content` empty until the reasoning is finished. + +The practical consequence is that a request must be allowed enough tokens to finish thinking *and* answer. If the limit is too low, the response comes back with `finish_reason` of `length`, an empty `content`, and the answer stranded mid-thought. A short question can easily spend a few hundred tokens reasoning before producing a one-sentence reply. + +!!! TIP + + If you see empty responses in the assistant, this is the first thing to check. Raise the token limit rather than assuming the model or the connection is broken. + +## Verify the Endpoint + +Confirm authentication is enforced. A request with no key, or the wrong key, must be rejected: + +```bash +curl -s -o /dev/null -w "%{http_code}\n" \ + -H "Content-Type: application/json" \ + http://127.0.0.1:8080/v1/chat/completions \ + -d '{"model":"gemma-4-26b","messages":[{"role":"user","content":"hi"}],"max_tokens":5}' +``` + +This should print `401`. + +!!! NOTE + + The `/v1/models` endpoint is deliberately not protected by the API key and will answer without one. It exposes only the model name. Inference endpoints require the key. + +Now make a real request: + +```bash +curl -s -H "Authorization: Bearer $(sudo cat /etc/llama-server/api-key)" \ + -H "Content-Type: application/json" \ + http://127.0.0.1:8080/v1/chat/completions \ + -d '{"model":"gemma-4-26b","messages":[{"role":"user","content":"In one sentence, what is Zeek used for?"}],"max_tokens":2000}' +``` + +The response includes a `timings` object reporting `prompt_per_second` and `predicted_per_second`, which is the simplest way to measure throughput on your own hardware. + +### Confirm Tool Calling + +The assistant relies on tools such as `query_events` and `ack_alerts` to reach data in your grid, so tool calling has to work. Send a request with a tool definition and confirm the model responds with a `tool_calls` entry rather than plain text: + +```bash +curl -s -H "Authorization: Bearer $(sudo cat /etc/llama-server/api-key)" \ + -H "Content-Type: application/json" \ + http://127.0.0.1:8080/v1/chat/completions -d '{ + "model": "gemma-4-26b", + "messages": [{"role":"user","content":"Are there alerts from source IP 10.1.2.3 in the last 24 hours? Use the available tools."}], + "tools": [{"type":"function","function":{ + "name":"query_events", + "description":"Query security events from the local Security Onion instance.", + "parameters":{"type":"object","properties":{"query":{"type":"string"},"range":{"type":"string"}},"required":["query"]} + }}], + "max_tokens": 2000 + }' +``` + +A correct result contains a `tool_calls` array naming `query_events` with populated arguments. + +## Performance Expectations + +The following were measured on the reference platform serving the `Q8_0` build with a 262,144 token context window. Your results will vary with hardware and quantization. + +| Measure | Result | +|---|---| +| Memory in use | ~32 GB of unified memory (26.9 GB of weights, the remainder cache and buffers) | +| Generation, short context | ~41 tokens/sec | +| Generation, ~193,000 token context | ~22 tokens/sec | +| Prompt processing, bulk | ~850 tokens/sec initially, ~228 tokens/sec averaged over a 193,000 token prompt | + +Two characteristics of this hardware are worth planning around. + +**Prompt processing slows as the context fills.** Bulk prompt processing starts around 850 tokens/sec but falls steadily with depth. A 193,139 token prompt took roughly 14 minutes to process before generation began, and generation itself ran at about half the speed it does on a short prompt. Large contexts are practical, but they are not fast. This is the main reason to set a generous Health Timeout on the adapter. + +**Memory is not the limiting factor here.** The model and a full-size context together use roughly a third of the available memory, which is why a 262,144 token window is practical on this machine. There is room for a larger model or additional concurrent slots if you need them. + +The context window is usable in practice, not just allocatable: in testing, a fact placed at the very beginning of a 193,139 token prompt was retrieved correctly when asked about at the end. + +!!! NOTE + + `LLAMA_ARG_N_PARALLEL` divides the context window among concurrent slots rather than adding capacity. Two slots on a 262,144 token window give each request 131,072 tokens. Raise the context size alongside the slot count if you need several analysts working at once with large contexts. + +## Troubleshooting + +Check the service log first: + +```bash +journalctl -u llama-server -n 100 --no-pager +``` + +**`failed to initialize ROCm: no ROCm-capable device is detected`**, followed by `no usable GPU found, --gpu-layers option will be ignored` + +The service account cannot reach the GPU devices. Confirm it is in the right groups and restart: + +```bash +id -nG _llama-server +sudo usermod -aG render,video _llama-server +sudo systemctl restart llama-server +``` + +Note that running `llama-cli --list-devices` from your own shell may still show the GPU even while the service cannot use it, because interactive sessions get access through a separate mechanism. Always confirm against the service log. + +**`request (N tokens) exceeds the available context size (262144 tokens)`** + +The prompt is larger than the configured window. Either reduce what is being sent or raise `LLAMA_ARG_CTX_SIZE`, keeping in mind the model's own 262,144 token limit. + +**Responses are empty and `finish_reason` is `length`** + +The token limit was reached while the model was still reasoning. See [Reasoning Output](#reasoning-output). + +**The service starts but the model never loads** + +The first start downloads the model, which is tens of gigabytes and can take a long time. Watch the cache directory grow: + +```bash +sudo du -sh /var/cache/llama-server +``` + +**Onion AI reports the adapter as unhealthy** + +Confirm the manager can reach the port, and raise the adapter's Health Timeout. A local model can take longer to respond than the default timeout allows, particularly on a large context. diff --git a/docs/logs.md b/docs/logs.md index bcae1c66..a1586151 100644 --- a/docs/logs.md +++ b/docs/logs.md @@ -1,17 +1,3 @@ -# Logs +# Logs Overview Once logs are generated by network sniffing processes or endpoints, where do they go? How are they parsed? How are they stored? That's what we'll discuss in this section. - -## Table of Contents - -- [Ingest](ingest.md) -- [Logstash](logstash.md) -- [Redis](redis.md) -- [Elasticsearch](elasticsearch.md) -- [ElastAlert](elastalert.md) -- [Data Fields](data-fields.md) -- [Alert Data Fields](alert-data-fields.md) -- [ElastAlert Fields](elastalert-fields.md) -- [Zeek Fields](zeek-fields.md) -- [Community ID](community-id.md) -- [Security Onion Console Logs](security-onion-console-logs.md) \ No newline at end of file diff --git a/docs/logstash.md b/docs/logstash.md index 1d09cdf2..94a8459b 100644 --- a/docs/logstash.md +++ b/docs/logstash.md @@ -71,7 +71,7 @@ In [SOC](security-onion-console.md), navigate to [Administration](administration custom/myfile.conf ``` -The configuration will be applied at the next 15-minute interval or you can apply it immediately by clicking the `SYNCHRONIZE Grid` button under the `Options` menu. +[Auto State Apply](salt.md#auto-state-apply) will apply the configuration within a few minutes. If you don't want to wait, click the `SYNCHRONIZE GRID` button under the `Options` menu. You can monitor events flowing through the output by running the following command on the manager: @@ -82,7 +82,7 @@ curl -s localhost:9600/_node/stats | jq .pipelines.manager ## Modified Event Forwarding -To forward events to an external destination AFTER they have traversed the Logstash pipelines (NOT ingest node pipelines), perform the same steps as above but instead of adding the reference for your Logstash output to the `manager` pipeline add it to `search` pipeline instead. The configuration will be applied at the next 15-minute interval or you can apply it immediately by clicking the `SYNCHRONIZE Grid` button under the `Options` menu. +To forward events to an external destination AFTER they have traversed the Logstash pipelines (NOT ingest node pipelines), perform the same steps as above but instead of adding the reference for your Logstash output to the `manager` pipeline add it to `search` pipeline instead. [Auto State Apply](salt.md#auto-state-apply) will apply the configuration within a few minutes. If you don't want to wait, click the `SYNCHRONIZE GRID` button under the `Options` menu. You can monitor events flowing through the output by running the following command on the search nodes: @@ -169,4 +169,4 @@ curl -k -XPUT -H 'Content-Type: application/json' https://localhost:9200/. \ No newline at end of file + For more information about Logstash, please see . diff --git a/docs/manager-of-managers.md b/docs/manager-of-managers.md index 15fe9861..49575172 100644 --- a/docs/manager-of-managers.md +++ b/docs/manager-of-managers.md @@ -4,9 +4,9 @@ This is an enterprise-level feature of Security Onion. Contact Security Onion Solutions, LLC via our website at for more information about purchasing a [Security Onion Pro](security-onion-pro.md) license with the necessary subgrid allocations to use this feature. -If you are an enterprise customer responsible for managing multiple grids, then you may benefit from the new Manager of Managers feature introduced with version 2.4.150! +If you are an enterprise customer responsible for managing multiple grids, then you may benefit from the Manager of Managers feature! -This feature allows you to elect a special Grid manager (MoM) to be configured with knowledge of remote grids (subgrids) allowing the MoM to reach out to the subgrid to query and/or modify Grid state, events, configuration, etc. +This feature allows you to elect a special grid manager (MoM) to be configured with knowledge of remote grids (subgrids) allowing the MoM to reach out to the subgrid to query and/or modify Grid state, events, configuration, etc. Appropriately privileged users logging into the MoM will be able to interact with those subgrids from a single user interface. A Subgrid selection list will be available in the top-right of the [SOC](security-onion-console.md) UI. Users can choose which Grid they would like to interact with. Additionally, fault states will automatically propagate up to the MoM user interface giving those users the quick updates of new fault states within subgrids, regardless of which subgrid is currently selected. If the subgrid list becomes disabled that will indicate the current screen does not support subgrid interaction. @@ -42,12 +42,12 @@ If a subgrid is expected to remain in a disconnected state for a longer period i Configuration of Manager of Managers requires two steps: -1. API Client: Creation of an API Client on each subgrid manager via the subgrid's [Connect](connect-api.md) Client screen +1. API Client: Creation of an API Client on each subgrid manager via the subgrid's API Client screen 2. Subgrid Config: Configuration of the subgrids on the MoM Configuration screen ### API Client -Create a new, dedicated [Connect](connect-api.md) Client on each subgrid and assign all permissions that the MoM will need to perform the desired functions. For example, if the MoM users will need complete management and visibility (effectively `superuser` access) then grant all permissions. However, if the MoM users will only be monitoring the subgrid health then the API Client permissions can be limited to `read` permission on specific resources, such as `Grid`, `nodes`, etc. +Create a new, dedicated API Client on each subgrid and assign all permissions that the MoM will need to perform the desired functions. For example, if the MoM users will need complete management and visibility (effectively `superuser` access) then grant all permissions. However, if the MoM users will only be monitoring the subgrid health then the API Client permissions can be limited to `read` permission on specific resources, such as `Grid`, `nodes`, etc. The API Client ID and Secret will be needed on the next step, so ensure those are recorded for each of the subgrids. @@ -55,7 +55,7 @@ While on the API Client screen, click the ⤓ icon to download the Certificate A !!! NOTE - Please be aware that any users in the MoM Grid will be able to connect to the subgrid using the permissions defined for the API client. For example, suppose that you create an API client ID in the subgrid called `supermom` and you grant it all permissions. Once the MoM is configured to connect to the subgrid as shown in the next section, then any users in the MoM Grid will connect to the subgrid as `supermom` and have all permissions to the subgrid regardless of whether the user has equivalent permissions in the MoM. + Starting in release 3.3.0, all MoM users that are not already superusers must be granted either the `subgrid-auditor` or `subgrid-superuser` role in order to interact with subgrids. The `subgrid-auditor` role will permit read operations (HTTP GET API requests), and `subgrid-superuser` will permit read and write operations (HTTP GET, POST, PUT, PATCH, and DELETE API requests) against the subgrids. These requests to the subgrid will all utilize the same subgrid API client credentials. As an example, a user with `subgrid-auditor` can see cases on the subgrid, but cannot modify them. Whereas a user with `subgrid-superuser` can modify cases on the subgrid, as well as manage subgrid users and their roles. ### Subgrid Config @@ -74,7 +74,7 @@ Once the subgrid API Client credentials are known that subgrid can then be added Add additional subgrids as your [Security Onion Pro](security-onion-pro.md) license allows, and then click the green checkmark to save the configuration. -The configuration will be applied at the next 15-minute interval or you can apply it immediately on the MoM Grid by clicking the `SYNCHRONIZE Grid` button under the `Options` menu. +[Auto State Apply](salt.md#auto-state-apply) should apply the configuration on the MoM Grid within a few minutes. ## Licensing @@ -82,4 +82,4 @@ The Manager of Managers feature requires that all involved grids have a valid [S The MoM Grid will require a special [Security Onion Pro](security-onion-pro.md) license with an allocation of subgrids encoded into the license that meets or exceeds the number of configured subgrids. Exceeding the licensed subgrid allocation will cause the MoM [SOC](security-onion-console.md) to show an "Exceeded" license state which will disable [Security Onion Pro](security-onion-pro.md) features. -Contact Security Onion Solutions, LLC via our website at for more information about purchasing a [Security Onion Pro](security-onion-pro.md) license with the appropriate subgrid allocations. \ No newline at end of file +Contact Security Onion Solutions, LLC via our website at for more information about purchasing a [Security Onion Pro](security-onion-pro.md) license with the appropriate subgrid allocations. diff --git a/docs/mcp-server.md b/docs/mcp-server.md index 4e2e9bab..bf103e78 100644 --- a/docs/mcp-server.md +++ b/docs/mcp-server.md @@ -6,7 +6,7 @@ The MCP server gives AI the ability to query alerts, playbooks, events, etc. whi !!! NOTE - The MCP server utilizes the Security Onion Connect API, which is an enterprise-level feature of Security Onion. Contact Security Onion Solutions, LLC via our website at for more information about purchasing a Security Onion Pro license to enable this feature. + The MCP server utilizes the Security Onion API, which is an enterprise-level feature of Security Onion. Contact Security Onion Solutions, LLC via our website at for more information about purchasing a Security Onion Pro license to enable this feature. ## Configuration @@ -16,15 +16,15 @@ See to get start !!! NOTE - A Connect API Client must be created in the Security Onion API Clients screen. The API Client should be granted sufficient permissions needed to perform the tasks that the LLM will need to execute. + A Security Onion API Client must be created in the Security Onion API Clients screen. The API Client should be granted sufficient permissions needed to perform the tasks that the LLM will need to execute. For example, if the LLM will be querying events and playbooks then the API Client will need *events/read*, *playbooks/read*, and *detections/read* permissions. ## Capabilities -As of 2.4.160, the MCP server currently supports the following operations: +The MCP server currently supports the following operations: - Query Events via Onion Query Language (OQL) - Query Playbooks -Future releases will likely bring more functionality, such as the ability to acknowledge alerts. \ No newline at end of file +Future releases will likely bring more functionality, such as the ability to acknowledge alerts. diff --git a/docs/netflow.md b/docs/netflow.md index 35ff7390..f831464a 100644 --- a/docs/netflow.md +++ b/docs/netflow.md @@ -10,7 +10,7 @@ First, add the Elastic integration for `NetFlow Records`. For more information about the `NetFlow Records` integration, please see . -1. Go to [Elastic Fleet](elastic-fleet.md), click the `Agent policies` tab, and then click the desired policy (for example `so-Grid-nodes_general`). +1. Go to [Elastic Fleet](elastic-fleet.md), click the `Agent policies` tab, and then click the desired policy (for example `so-grid-nodes_general`). 2. Click the `Add integration` button. 3. Search for `netflow` and then click on the `NetFlow Records` integration. 4. The Elastic Integration page will show an overview of the NetFlow Integration. Review all information on the page and then click the `Add NetFlow Records` button. @@ -29,7 +29,7 @@ Next, allow the traffic from the NetFlow exporter through the firewall to the Ne 3. On the left side, go to `firewall`, select `hostgroups`, and click the `customhostgroup0` group. On the right side, enter the IP address of the NetFlow exporter and click the checkmark to save. 4. On the left side, go to `firewall`, select `portgroups`, select the `customportgroup0` group, and then click `udp`. On the right side, enter your desired NetFlow listener port (2055 by default) and click the checkmark to save. 5. On the left side, go to `firewall`, select `role`, and then select the node type that will receive the NetFlow records. Then drill into `chain` --> `INPUT` --> `hostgroups` --> `customhostgroup0` --> `portgroups`. On the right side, enter `customportgroup0` and click the checkmark to save. -6. If you would like to apply the rules immediately, click the `SYNCHRONIZE Grid` button under the `Options` menu at the top of the page. +6. If you would like to apply the rules immediately, click the `SYNCHRONIZE GRID` button under the `Options` menu at the top of the page. ## NetFlow dashboard diff --git a/docs/network-installation.md b/docs/network-installation.md index cbb7f00e..2b0e8573 100644 --- a/docs/network-installation.md +++ b/docs/network-installation.md @@ -2,7 +2,7 @@ !!! WARNING - Network installations are NOT supported and should only be used as a last resort in case there is some reason you can't use our official Security Onion images as shown in the [installation](installation.md) section. + Network installations are NOT supported and should only be used as a last resort in case there is some reason you can't use our official Security Onion images as shown in the [Installation](installation.md) section. Our official Security Onion images (ISO image and cloud images) are the ONLY supported installation method and you should use them if any of the following apply to you: @@ -20,7 +20,7 @@ Our official Security Onion images take care of partitioning for you. However, i ### Minimum Storage -As the [hardware](hardware.md) section mentions, the MINIMUM requirement is 200GB storage. This is to allow 100GB for `/nsm` and 100GB for the rest of `/`. +As the [Hardware](hardware.md) section mentions, the MINIMUM requirement is 200GB storage. This is to allow 100GB for `/nsm` and 100GB for the rest of `/`. ### LVM @@ -80,9 +80,9 @@ If you understand all of the warnings above and still want to perform a network - Download our repo and start the Setup process: ``` - git clone -b 2.4/main https://github.com/Security-Onion-Solutions/securityonion + git clone -b 3/main https://github.com/Security-Onion-Solutions/securityonion cd securityonion sudo bash so-setup-network ``` -- Proceed to the [Configuration](configuration.md) section. \ No newline at end of file +- Proceed to the [Configuration](configuration.md) section. diff --git a/docs/network-visibility.md b/docs/network-visibility.md index 8bf4e12b..b7203e73 100644 --- a/docs/network-visibility.md +++ b/docs/network-visibility.md @@ -1,15 +1,5 @@ -# Network Visibility +# Network Visibility Overview -When you log into [SOC](security-onion-console.md), you may see alerts from [Suricata](suricata.md) or [IDH](idh.md), protocol metadata logs from [Zeek](zeek.md) or [Suricata](suricata.md), file analysis logs from [Strelka](strelka.md), or full packet capture from [Suricata](suricata.md). How is that data generated and stored? This section covers the various processes that Security Onion uses to analyze and log network traffic. +When you log into [Security Onion Console](security-onion-console.md), you may see alerts from [Suricata](suricata.md) or [IDH](idh.md), protocol metadata logs from [Zeek](zeek.md) or [Suricata](suricata.md), file analysis logs from [Strelka](strelka.md), or full packet capture from [Suricata](suricata.md). How is that data generated and stored? This section covers the various processes that Security Onion uses to analyze and log network traffic. ![Image](images/diagrams/sniffing.png) - -## Table of Contents - -- [AF-PACKET](af-packet.md) -- [BPF](bpf.md) -- [Full Packet Capture](full-packet-capture.md) -- [Suricata](suricata.md) -- [Zeek](zeek.md) -- [Strelka](strelka.md) -- [IDH](idh.md) \ No newline at end of file diff --git a/docs/new-disk.md b/docs/new-disk.md index 53271ca9..6eb06611 100644 --- a/docs/new-disk.md +++ b/docs/new-disk.md @@ -16,9 +16,9 @@ If your disk is partitioned using LVM (the default for our Security Onion ISO im ### LVM Example -For a simple LVM example, suppose that you have installed our Security Onion ISO image in a virtual machine (VM) using a virtualization solution like [proxmox](proxmox.md) that allows you to increase storage on the fly. Our Security Onion ISO image automatically uses LVM and should use the XFS filesystem for `/nsm`, so if you want to add space to `/nsm` here is a brief overview of the steps you would use. +For a simple LVM example, suppose that you have installed our Security Onion ISO image in a virtual machine (VM) using a virtualization solution like [Proxmox](proxmox.md) that allows you to increase storage on the fly. Our Security Onion ISO image automatically uses LVM and should use the XFS filesystem for `/nsm`, so if you want to add space to `/nsm` here is a brief overview of the steps you would use. -First, expand the storage. If using [proxmox](proxmox.md), select the VM, select `Hardware`, select the `Hard Disk`, click `Disk Action`, click `Resize`, enter the amount that you would like to add to the existing disk size, and then click the `Resize disk` button. +First, expand the storage. If using [Proxmox](proxmox.md), select the VM, select `Hardware`, select the `Hard Disk`, click `Disk Action`, click `Resize`, enter the amount that you would like to add to the existing disk size, and then click the `Resize disk` button. Next, log into the VM, become root, and use a partition tool like `cfdisk` to resize the LVM partition to take advantage of the additional free space: @@ -105,4 +105,4 @@ df -Th If you aren't using LVM but you need to expand your `/nsm` partition, then you can mount a separate physical drive directly to `/nsm`. If doing this after installation, you will need to stop services, move the data to the new drive, and then restart services. -A variation on this method is to make `/nsm` a symbolic link to the new logging location. Certain services like AppArmor may need special configuration to handle the symlink. \ No newline at end of file +A variation on this method is to make `/nsm` a symbolic link to the new logging location. Certain services like AppArmor may need special configuration to handle the symlink. diff --git a/docs/nids.md b/docs/nids.md index 5e22fd79..693eba02 100644 --- a/docs/nids.md +++ b/docs/nids.md @@ -43,7 +43,7 @@ If a detection would be matched by both an enable and disable regex, it is enabl Enable and disable operations that are based on regex patterns are actioned during the daily rule update. If you have made a change to the regex patterns and would like to have it implemented more immediately: -- Under Grid Configuration, click the `SYNCHRONIZE Grid` button and wait about 5 minutes for it to complete. +- Under Grid Configuration, click the `SYNCHRONIZE GRID` button and wait about 5 minutes for it to complete. - Navigate to [Detections](detections.md), click the Options menu, select [Suricata](suricata.md) in the dropdown menu, click the `FULL UPDATE` button, and then wait for it to complete. - Refresh the [Detections](detections.md) page and you should see the relevant rule statuses have changed. @@ -202,7 +202,7 @@ To add a new NIDS rule, go to the main [Detections](detections.md) page and clic 1. Click the Language drop-down and select `Suricata`. 2. Optionally specify a license. 3. Add the signature. -4. Click the `CREATE` button and the detection should deploy to your Grid at the next 15-minute cycle. +4. Click the `CREATE` button and [Auto State Apply](salt.md#auto-state-apply) should deploy the detection to your grid within a few minutes. ![Image](images/59_detection_create.png) @@ -222,8 +222,8 @@ You can enable external access to NIDS rules managed by [Detections](detections. - At the top of the page, click the `Options` menu and then enable the `Show advanced settings` option. - Navigate to Nginx --> config --> external_suricata. - On the right side of the page, change the value to `true` and then click the checkmark to save the new setting. -- You can wait for the next Grid update or click the `SYNCHRONIZE Grid` button under Options. -- Once the Grid is fully synchronized, the Manager should listen on port 7789 for https connections from hosts defined in the `external_suricata` host group. +- You can wait for the next Grid update or click the `SYNCHRONIZE GRID` button under Options. +- Once the grid is fully synchronized, the Manager should listen on port 7789 for https connections from hosts defined in the `external_suricata` host group. ## Configuring Rulesets @@ -236,7 +236,7 @@ There are two configuration profiles: If your system is in Airgap mode, the Airgap configuration profile will automatically be used - otherwise the default is in use. -Within this configuration, you can enable additional rulesets, add custom rulesets, or disable existing ones. When you save a ruleset configuration change and apply the SOC state, Security Onion will detect the change and automatically sync all configured rulesets within 15 minutes. +Within this configuration, you can enable additional rulesets, add custom rulesets, or disable existing ones. When you save a ruleset configuration change and apply the SOC state, Security Onion will detect the change and sync all configured rulesets. Once the rules have been written on the manager node, [Auto State Apply](salt.md#auto-state-apply) deploys them to the sensors within a few minutes. OISF-maintained list of Suricata-compatible rulesets: @@ -379,7 +379,7 @@ For Airgap deployments using ET Pro (commercial) rules, you must manually transf - **Apply configuration and sync** - Save the configuration and apply the SOC state. Then either wait for the next automatic sync (up to 15 minutes) or trigger a manual sync: + Save the configuration and apply the SOC state. Then either wait for the next automatic sync or trigger a manual sync: - Navigate to [Detections](detections.md) - Click Options menu @@ -484,7 +484,7 @@ To resolve this block, use the following procedure: Suricata ruleset sync is blocked until this file is removed. **CRITICAL** Make sure that you have manually added any custom Suricata rulesets via SOC config before removing this file - review the documentation - for more details: + for more details: Custom so-rule-update detected (hash: 207d8918a2d963bb7dcc0f1ebf28d6f7b5778019fedf0cc36d5d0850cbd8a529) ET Pro code found: YOUR_LICENSE_KEY ``` diff --git a/docs/notifications.md b/docs/notifications.md index a018bd77..2b15f9ba 100644 --- a/docs/notifications.md +++ b/docs/notifications.md @@ -4,7 +4,7 @@ This is an enterprise-level feature of Security Onion. Contact Security Onion Solutions, LLC via our website at for more information about purchasing a Security Onion Pro license to enable this feature. -The [Detections](detections.md) module, specifically [Sigma](sigma.md) rules, can be enabled to send outbound notifications upon an alert being created. By default, no outbound notifications are enabled in a Security Onion installation. However, with the Pro license applied to a Grid, notifications can be quickly configured via the Configuration screen. +The [Detections](detections.md) module, specifically [Sigma](sigma.md) rules, can be enabled to send outbound notifications upon an alert being created. By default, no outbound notifications are enabled in a Security Onion installation. However, with the Pro license applied to a grid, notifications can be quickly configured via the Configuration screen. ## Configuration @@ -69,7 +69,7 @@ Important! After activating (or removing) an alerter from this setting, the [Ela ### Severity-Based Notifications -The instructions above setup the default notification settings, for all outbound notifications. However, as of Security Onion 2.4.100, notification settings can be customized for higher level severities. Severities are specified in Sigma [Detections](detections.md). +The instructions above setup the default notification settings, for all outbound notifications. However, notification settings can be customized for higher level severities. Severities are specified in Sigma [Detections](detections.md). Severity levels progress as follows, starting with the lowest, least significant severity: @@ -88,7 +88,7 @@ If notification settings are not specified for a particular severity level then ### User-Defined Notifications -As of Security Onion 2.4.100, individual Sigma detections can be tagged to change the detection's alerting behavior. The tags are set inside the detection source. Tag details are defined below: +Individual Sigma detections can be tagged to change the detection's alerting behavior. The tags are set inside the detection source. Tag details are defined below: - `so.notification`: When this tag is present inside of a Sigma tag list, the detection will only perform outbound notifications. It will not add an alert to the SOC Alerts screen. - `so.alerters.customAlerters`: When this tag is present inside of a Sigma tag list, the detection will perform notifications for an alternate set of ElastAlert 2 alerters. More information on how to choose these alerters is provided below. @@ -108,7 +108,7 @@ Example: title: Security Onion - Grid Node Login Failure (SSH) (copy) id: 0c880a39-f2cc-4e80-af26-eb08e2fe4b0a status: experimental -description: Detects when a user fails to login to a Grid node via SSH. Review associated logs for username and source IP. +description: Detects when a user fails to login to a grid node via SSH. Review associated logs for username and source IP. author: Security Onion Solutions date: 2024/08/27 logsource: @@ -118,7 +118,7 @@ detection: selection: event.outcome: failure process.name: sshd - tags|contains: so-Grid-node + tags|contains: so-grid-node filter: system.auth.ssh.method: '*' condition: selection and not filter @@ -159,10 +159,10 @@ alert_text_args: ["log.id.uid", "source.ip", "source.port", "destination.ip", "d In order for alerters and parameters to take effect, multiple synchronizations must occur. These are done automatically on a set schedule, but it is possible to force them earlier, if needed. Specifically, the following must take place for the changes to be applied to the ElastAlert 2 rules: 1. Changes are saved in Configuration screen by the SOC Admin. -2. Configuration is synchronized across the Grid. To manually force a Grid sync, go to the Configuration screen, open the `Options` dropdown at the top, and click `Synchronize`. +2. Configuration is synchronized across the grid. To manually force a grid sync, go to the Configuration screen, open the `Options` dropdown at the top, and click `Synchronize`. 3. Sigma Detection edits are saved, such as adding the user-defined notification tags, or changing the severity. 4. Sigma Detections are synchronized. Click Full Synchronize for ElastAlert rules, or to force a single detection sync go to the Detection Source tab, make an edit to the source, and click `Update`. !!! NOTE - It may take a minute or two for the ElastAlert 2 process to detect the changed rules, and then another few minutes for ElastAlert 2 to run that rule. \ No newline at end of file + It may take a minute or two for the ElastAlert 2 process to detect the changed rules, and then another few minutes for ElastAlert 2 to run that rule. diff --git a/docs/oidc.md b/docs/oidc.md index b75e0b6e..b85b11bd 100644 --- a/docs/oidc.md +++ b/docs/oidc.md @@ -1,6 +1,6 @@ # OpenID Connect (OIDC) -Starting with Security Onion version 2.4.30, SOC supports single sign-on (SSO) authentication via OpenID Connect (OIDC) to one of several OIDC-compatible identity providers. For example, users can login to Security Onion using an Active Directory user, a GitHub user, a Google account, an Auth0 account, etc. Only one OIDC provider can be active at a time. +SOC supports single sign-on (SSO) authentication via OpenID Connect (OIDC) to one of several OIDC-compatible identity providers. For example, users can login to Security Onion using an Active Directory user, a GitHub user, a Google account, an Auth0 account, etc. Only one OIDC provider can be active at a time. !!! NOTE @@ -18,7 +18,7 @@ Starting with Security Onion version 2.4.30, SOC supports single sign-on (SSO) a OIDC configuration can be complex and we recommend taking advantage of the official Security Onion support team. Note that purchases of a Security Onion license include some level of support. This will help avoid time-consuming problems that can occur when configuring OIDC. -The first step in configuration OIDC is to determine which provider the Grid will use, and collecting the required configuration inputs necessary for that specific provider. +The first step in configuration OIDC is to determine which provider the grid will use, and collecting the required configuration inputs necessary for that specific provider. Next, in Security Onion Console, while logged in as an administrator, navigate to the Administration -> Configuration screen and enter `oidc` into the filter field. Then click the *Expand All* icon. @@ -138,7 +138,7 @@ Generate a new client secret under the Ping `Client ID` field. Copy the generate Locate the `client_secret` setting in the SOC configuration screen back on the SOC browser tab. Specify the above client secret for this setting. -On the Ping console browser tab, under the configuration tab, expand the URLs section, near the top. Copy and paste the three following URLs into the appopriate SOC configuration screen settings: +On the Ping console browser tab, under the configuration tab, expand the URLs section, near the top. Copy and paste the three following URLs into the appropriate SOC configuration screen settings: - Authorization URL -> auth_url - Issuer -> issuer_url @@ -162,7 +162,7 @@ Finally, enable OIDC by locating the `enabled` setting in the SOC configuration !!! NOTE - Do not enable OIDC until all required configuration settings have been entered and double-checked for accuracy. Once enabled the backend system will automatically synchronize the settings across the Grid, typically within 15 minutes. If some settings are incorrect or missing the backend authentication services could be left in an error state and make it impossible to fix via the Configuration screen, as the SOC UI may no longer be accessible. If this occurs an SSH session will be required to access the underlying configuration files on the manager node. Contact support for assistance if needed. + Do not enable OIDC until all required configuration settings have been entered and double-checked for accuracy. Once enabled the backend system will automatically synchronize the settings across the grid, typically within a few minutes. If some settings are incorrect or missing the backend authentication services could be left in an error state and make it impossible to fix via the Configuration screen, as the SOC UI may no longer be accessible. If this occurs an SSH session will be required to access the underlying configuration files on the manager node. Contact support for assistance if needed. !!! WARNING @@ -174,9 +174,9 @@ Upon the first login via OIDC the user will likely be returned back to the login ## Roles -When a new OIDC user logs into SOC, that user will not be assigned any roles. This greatly limits what functions the user will be capable of performing within SOC. For example, new users will be unable to see any alerts, hunt for events, view dashboard data, view or create cases, manage the Grid, or view other users. Attempting to view those role-protected screens will result in an error message. +When a new OIDC user logs into SOC, that user will not be assigned any roles, unless a [default system role](rbac.md#default-role-assignment) has been configured. This greatly limits what functions the user will be capable of performing within SOC. For example, new users will be unable to see any alerts, hunt for events, view dashboard data, view or create cases, manage the grid, or view other users. Attempting to view those role-protected screens will result in an error message. -An administrator will need to login to SOC and assign roles to OIDC users via the Adminstration -> Users screen. This is a one time operation, per user. +If a default system role is not configured, an administrator will need to login to SOC and assign roles to OIDC users via the Adminstration -> Users screen. This is a one time operation, per user. ## Managing OIDC Users @@ -206,4 +206,4 @@ When all local authentication methods have been disabled, users will have no sec ## External Tools -Tools included with Security Onion, but provided by other vendors, will not utilize SOC single sign-on. This includes tools such as InfluxDB, Kibana and other Elastic-provided tools. If users need to access these tools the password authentication method must be enabled and a local password setup. The users can then login to those tools using their SSO email address and the local SOC password. \ No newline at end of file +Tools included with Security Onion, but provided by other vendors, will not utilize SOC single sign-on. This includes tools such as InfluxDB, Kibana and other Elastic-provided tools. If users need to access these tools the password authentication method must be enabled and a local password setup. The users can then login to those tools using their SSO email address and the local SOC password. diff --git a/docs/onion-ai.md b/docs/onion-ai.md index db992f39..d5d621b8 100644 --- a/docs/onion-ai.md +++ b/docs/onion-ai.md @@ -1,27 +1,125 @@ # Onion AI !!! NOTE - + This is an enterprise-level feature of Security Onion. Contact Security Onion Solutions, LLC via our website at for more information about purchasing a [Security Onion Pro](security-onion-pro.md) license. -The Onion AI Assistant is your personal AI helper designed to assist you with a variety of tasks and provide information on demand. Several tools have been made available as you interact with the assistant so it can access up-to-date information and resources. +!!! NOTE + + Onion AI is disabled by default. To enable it for your Security Onion Pro deployment, go to the Administration --> Configuration page and navigate to soc --> config --> server --> client --> assistant --> enabled. + +The Onion AI Assistant is your personal AI helper designed to assist you with a variety of tasks and provide information on demand. We support accessing LLMs from a variety of sources. Several tools have been made available as you interact with the assistant so it can access up-to-date information and resources. + +## Adapters + +There are a variety of adapters available to connect to different LLM providers. When configuring an adapter, you're instructing SOC how to connect to an LLM provider. You configure the models separately in [Available Models](#available-models) and tell each model which adapter to use. This allows you to connect to multiple providers at the same time and choose which models you want to use from each provider. The available adapters are: + +- **SOAI**: The Security Onion AI adapter connects to our own hosted gateway and provides access to a variety of models. This is the default adapter, and is hosted in the cloud with an allotted number of credits per Security Onion Pro licensed grids. +- **Gemini**: Google's Gemini models can be accessed through this adapter. You can connect through either the Gemini Developer API or the Vertex API. These models are hosted in Google's cloud and require the use of your own Google Cloud key, or Gemini API key. +- **OpenAI Responses**: This adapter can connect to the newest OpenAI compatible APIs that support the Responses protocol. This adapter allows Security Onion grids to connect to locally-hosted LLMs. +- **OpenAI Chat**: This adapter can connect to any OpenAI Chat compatible APIs, an older protocol still supported by many AI providers. This adapter allows Security Onion grids to connect to locally-hosted LLMs. + +Most features are supported across all adapters, but capabilities may change depending on the model or provider you connect to. Credits are unique to SOAI, it's the only adapter that responds with a credit balance. Multiple adapters can be configured at the same time. -## Credits +### Configuration + +A fresh install will come with the SOAI adapter pre-configured and ready to use. To connect to other providers or modify the SOAI adapter, go to Administration --> Configuration page, be sure "Show advanced settings" is turned on in the Options at the top of the page, and check under soc --> config --> server --> modules --> assistant --> adapters. Here you can add as many adapters as you need. Each adapter defines how to connect to a provider. Different adapters have different configuration requirements. Every adapter needs a unique name, the protocol it uses, and a Health Timeout measured in seconds. Below is a table of which fields are required for which adapters: + +| | Adapter Name | Protocol | API Url | API Key | Service Account JSON | Service Account Location | Health Timeout Seconds | +|---|---|---|---|---|---|---|---| +| SOAI | R | R | R | -- | -- | -- | R | +| Gemini | R | R | -- | O | O | O | R | +| OpenAI Responses | R | R | R | O | -- | -- | R | +| OpenAI Chat | R | R | R | O | -- | -- | R | + +R = Required, O = Optional, -- = Not Used + +### SOAI + +This adapter should not require any configuration, aside from the API Url, which will be different for Security Onion Pro customers located in different regions. To change the SOAI API Url, go to the Administration --> Configuration page and navigate to soc --> config --> server --> modules --> assistant --> adapters. Select the SOAI adapter and update the API Url field. If your license is setup to use an alternate cloud region, the correct API Url will be included with your Security Onion license key. If an alternate URL is not specified, use the default API Url. + +Any additionally supplied fields are ignored. Generally at most one instance of this adapter should be configured at a time. + +#### Credits + +Using models over the SOAI adapter will consume credits from your Security Onion Pro license. Credits are only applicable to the SOAI adapter. If you use a local model or another external provider (such as Gemini or OpenAI), SOAI credits are not consumed. The Security Onion Pro license includes an initial amount of credits to get started. For long term usage planning contact your Security Onion account representative. They will assist with estimating credit usage rates as well as the provisioning of additional credits. Credits are consumed based on the number of tokens used in the conversation including user input, assistant output, and tool usage. Your organization's balance can be viewed at the top right of the assistant page or on the management page under Administration --> AI Metrics. -## Available Tools +### Gemini -The assistant currently runs in the cloud. In order to reference local information, SOC makes the following tools available to the assistant: +Security Onion supports connecting to Google's Gemini models through either the Gemini Developer API or the Vertex API. + +To connect over the Gemini Developer API, you will need to generate an API Key in the Google Cloud Console or Google AI Studio, and provide it in the API Key field. To connect over the Vertex API, you will need to create a Service Account with the appropriate permissions for the models you would like to access, generate a JSON key for that account, paste the JSON into the Service Account JSON field, and you must also specify the Service Account Location. Some Gemini models may only be available in certain locations, such as `global`. The API Url field is not used for Gemini connections. + +If an API Key and Service Account JSON + Service Account Location are both provided, SOC will attempt to connect using the Service Account and the API Key will be ignored. + +This adapter will not use any credits from your Security Onion Pro license, but Google will charge you based on their pricing. Security Onion Solutions, LLC does not provide support for Google Gemini, aside from assisting with the adapter configuration referenced on this page. + +### OpenAI Responses - - **query_events**: This read-only tool allows the assistant to query security events from your local Security Onion instance similar to how you would use the [Hunt](hunt.md) page. - - **get_playbooks**: When the assistant uses this read-only tool, SOC will gather the playbooks, execute their queries, and return all the data ready for analysis. - - **ack_alerts**: The assistant can use this tool to query and acknowledge [Alerts](alerts.md) in your local Security Onion instance. - - **escalate_alerts**: Similar to the ack tool, this tool allows the assistant to query for alerts and escalate them to a new case. +The *OpenAI Responses* adapter can connect to the newest OpenAI compatible APIs that support the Responses protocol. To connect, you will need to provide an API Url. The API Url should be the base url for the provider you are connecting to, for example `https://api.openai.com/v1/`. An API Key may or may not be required depending on the provider you are connecting to. If you're unsure if your provider supports the Responses protocol, check with their support or try connecting with this adapter. If it doesn't work, you can try the *OpenAI Chat* adapter which supports an older protocol that is still widely used. !!! TIP - - Currently this feature requires an internet connection. Local model support is coming soon! + + Some providers require URL parameters passed along with the API calls. For example, Azure requires an `api-version` parameter. To support this, starting with Security Onion 3.1.0, you can append the parameter(s) to the API Url field. For example: + + ```text + https://api.someprovider.invalid/v1/?api-version=2024-08-01-preview + ``` + +### OpenAI Chat + +The *OpenAI Chat* adapter can connect to any OpenAI compatible API that supports the Chat protocol. To connect, you will need to provide an API Url. The API Url should be the base url for the provider you are connecting to, for example `https://api.openai.com/v1/`. An API Key may or may not be required depending on the provider you are connecting to. + +!!! TIP + + Some providers require URL parameters passed along with the API calls. To support this, starting with Security Onion 3.1.0, you can append the parameter(s) to the API Url field. For example: + + ```text + https://api.someprovider.invalid/v1/?api-version=2024-08-01-preview + ``` + +## Available Models + +In order to control what models are available inside SOC, individual models must be configured. To configure a model, go to the Administration --> Configuration page, be sure "Show advanced settings" is turned on in the Options at the top of the page, and check under soc --> config --> server --> client --> assistant --> availableModels. Here you can add as many models as you need. Each model needs a unique name, the adapter it uses, and a model identifier which tells the adapter which model to connect to on the provider. + +!!! NOTE + + If a provider offers models that aren't listed in availableModels, they will not be accessible within SOC. + +## Local Model Considerations + +Security Onion now supports local models through any OpenAI-compatible endpoint. Because the assistant relies on large context windows, the minimum recommended context length is 128k tokens. The following open-source models have been tested with OnionAI: + +- **GPT OSS 120B** -- (US) Fast inference but limited to a 128k context window. Accuracy is fair. +- **Kimi 2.5** -- (China) A capable model with average accuracy and a 256k context window sufficient for most tasks. Note that this model requires significant VRAM to maintain performance. +- **GLM 5** -- (China) Average accuracy with a 200k context window. +- **Qwen 3.5** -- (China) Average accuracy with a 200k context window. + +!!! NOTE + + Local models will not match the performance of proprietary foundational models such as those from Anthropic, Google, or OpenAI. + +## Hosting Local Models + +Hosting your own models requires powerful and expensive hardware. For beginners we recommend using a tool such as LM Studio. The models listed above are large, and **you need at least 96GB of VRAM** to host them locally. Smaller mixture-of-experts models can run in considerably less memory while still offering a context window large enough for the assistant; see [Local LLM Hosting](local-llm.md) for a tested walkthrough on a single machine. The speed and accuracy of OnionAI when hosted locally is based on the hardware that you are using. For the most accurate results we recommend using credits with OnionAI. + +## Available Tools + +The assistant currently runs in the cloud. In order to reference local information, SOC makes the following tools available to the assistant: + +- **query_events**: This read-only tool allows the assistant to query security events from your local Security Onion instance similar to how you would use the [Hunt](hunt.md) page. +- **query_cases**: This tool allows the assistant to query cases from your local Security Onion instance similar to how you would use the [Cases](cases.md) page. +- **query_detections**: This tool allows the assistant to query detections from your local Security Onion instance similar to how you would use the [Detections](detections.md) page. +- **get_playbooks**: When the assistant uses this read-only tool, SOC will gather the playbooks, execute their queries, and return all the data ready for analysis. This tool is read-only. +- **ack_alerts**: The assistant can use this tool to query and acknowledge [Alerts](alerts.md) in your local Security Onion instance. +- **escalate_alerts**: Similar to the ack tool, this tool allows the assistant to query for alerts and escalate them to a new case. +- **create_detection**: The assistant will use this tool to create new detections in your local Security Onion instance. +- **add_overrides**: This tool allows the assistant to tune existing detections by adding overrides to them. +- **toggle_detections**: The assistant can use this tool to enable or disable existing detections in your local Security Onion instance. +- **update_detection_content**: If a detection needs its content updated, the assistant can use this tool to update the content and keep the same metadata, overrides, and enabled status. +- **update_overrides**: This tool allows the assistant to update existing overrides for detections. ### Permissions @@ -31,10 +129,12 @@ When the assistant requests any of these tools, the user will be prompted to app The following read-only tools can be used by the assistant without requiring explicit user approval: - - query_events - - get_playbooks +- query_events +- query_cases +- query_detections +- get_playbooks -By default, permission must be granted for each tool request. However, users can enable auto-approval for read-only tools by toggling the slider in the Options dropdown at the top of the assistant page. With the option enabled, read-only tools will auto approve, while tools that modify data (acknowledge and escalate alerts) will still request explicit user approval. +By default, permission must be granted for each tool request. However, users can enable auto-approval for read-only tools by toggling the slider in the Options dropdown at the top of the assistant page. With the option enabled, read-only tools will auto approve, while tools that modify data will still request explicit user approval. !!! NOTE @@ -54,4 +154,64 @@ Your system prompt addendum will be added after Security Onion's default system Superusers can review token usage and conversation history for all users by going to Administration --> AI Metrics. This page provides usage statistics for a given date range. The page starts with a table of usage by user. Clicking a user's binoculars icon on the right hand side will show any sessions the user interacted with during the selected date range, even deleted sessions. Clicking on a session's binoculars icon will show the full conversation. Administrators can adjust who has permissions via RBAC roles. -To provide an accurate history, deleted sessions are retained on the metrics page even after being deleted by the user. \ No newline at end of file +To provide an accurate history, deleted sessions are retained on the metrics page even after being deleted by the user. + +## Memory + +OnionAI can utilize its memory system to retain and reference facts disclosed by users when chatting with the assistant. Memory works by scanning sessions in the background looking for new facts and storing them in a database. The memory is then used to provide contextual information to the assistant when it responds to a user's message. + +### How Memory Works + +The memory system scans sessions in the background extracting, embedding, reconciling, and saving memories to later be referenced when chatting with Onion AI. + +First, extraction. The selected sessions have their transcripts given to a Memory agent that parses out useful facts. The agent also identifies the scope of the fact: is it specific to the user who owns the session we just scanned or is it applicable to the entire organization. + +These facts are then embedded using an Embed agent. This enables SOC to compare memories and is the heart of how memory works. + +!!! NOTE + + The embedding process is model specific. Memories embedded by one model cannot be accurately compared to memories embedded by any other model. Changing the model used for embedding will render all your previous memories unusable. They will not be deleted and will be accessible if the model is reverted back. + +Once embedded, the facts are compared against existing memories on the same topics in a step called Reconcilation. A Reconcile agent will decide how the facts should be added, merged, replaced or removed. The reconcile agent's recommendations are validated before being executed to ensure that user defined memories aren't modified and that the session owner's permissions are respected. + +If any facts were rewritten during reconcilation, they are re-embedded before being stored in postgres. + +Finally the session is updated indicating that it has been scanned. If new messages are sent in a previously scanned session, memory scans will find and scan only the new messages. + +When a user sends a message to the assistant, the memory system will embed the message about to be sent and check for any similar memories. Memories are added to the conversation by being added to the end of the prompt. + +### Configuring Memory + +| Name | Default Value | Description | +|---|---|---| +| `useMemory` | `true` | Enable memory use when sending a message. | +| `useMemoryScanner` | `false` | Enables the scanning of historical sessions. | +| `dontScanBefore` | `""` | A date in RFC3339 format (2026-08-31T22:05:48Z) to use as a cutoff point for memory scans. | +| `memoryScanIntervalSeconds` | `300` | How long between scans the memory scanner waits before scanning again. | +| `memoryProximityThreshold` | `0.8` | A similarity threshold that determines how similar a fact must be to a previous memory for it to be considered "similar." | +| `messageProximityThreshold` | `0.5` | A similarity threshold that determines how similar a message from a user must be to a stored memory for it to be considered "similar." | +| `maxUserMemoriesToReconcile` | `20` | The maximum number of user-specific memories to reconcile at once. | +| `maxGlobalMemoriesToReconcile` | `20` | The maximum number of global memories to reconcile at once. | +| `maxUserMemoriesToInclude` | `5` | The maximum number of user-specific memories to include when sending a message. | +| `maxGlobalMemoriesToInclude` | `5` | The maximum number of global memories to include when sending a message. | +| `memoryModel` | `gemma@SOAI` | The model used to extract facts from sessions. | +| `memoryPersona` | `""` | Special instructions for the memory agent included in the prompt. | +| `embedModel` | `amazon.titan-embed-text-v2@SOAI` | The model used to embed facts. | +| `reconcileModel` | `gemma@SOAI` | The model used to reconcile facts from sessions with existing memories. | +| `reconcilePersona` | `""` | Special instructions for the reconcile agent included in the prompt. | +| `toolUseTurnAttempts` | `12` | When the assistant requests a read-only tool, SOC can approve it automatically. Because the approval can fire before the original request has finished being written to Elasticsearch, SOC will retry the approval up to this many times before giving up. | +| `toolUseTurnDelayMs` | `175` | The time to wait between auto-approval attempts. Together with the attempts setting, this defines the total grace period SOC allows for the tool request to become available. | +| `maxMemoryRetries` | `2` | The maximum number of times SOC will attempt to extract memories from sessions before marking the session to be ignored. Increasing this value may retry sessions that haven't been attempted in a long time. | + +Memory and the Memory Scanner may be enabled independently. Disabling memory will stop applying memories to prompts on outgoing messages. Disabling the memory scanner will prevent the scanner from extracting memories from previous assistant sessions. The memory scanner marks sessions as it extracts facts from them so that they are not re-scanned in future scans. If a memory scan takes longer than the interval between scans, then at most 1 scan will queue up for processing and it'll begin again immediately after the previous scan finishes. + +### How to Write a Good Memory + +When entering memories in manually in the Agent Studio, it's important to understand that how the memory is written affects how it'll be used by the memory system. + +1. **One fact per memory.** If a memory has multiple facts then matching is harder to accomplish resulting in the memory being accessed less. +1. **Make each memory self-contained.** It will be read out of context, alone, possibly months later in a conversation about something else. Every memory should make sense to someone who has only that sentence. +1. **Declarative facts, not instructions.** If you want to give an instruction, then use a prompt instead. +1. **Present tense, absolute references, durable phrasing.** A memory is read long after it's written and is never automatically updated, so describe how things are rather than how they changed. Prefer "PCAP retention is 90 days (set August 2026)" over "We recently changed retention to 90 days." + +To help track your most successful memories, the Agent Studio presents how many times a memory has been accessed and when was the last time it was referenced. diff --git a/docs/opnsense.md b/docs/opnsense.md index 3be57a7c..09bb465e 100644 --- a/docs/opnsense.md +++ b/docs/opnsense.md @@ -4,7 +4,7 @@ OPNsense is a free and open firewall that can be found at diff --git a/docs/osquery-manager.md b/docs/osquery-manager.md index 4369ea64..33a078d0 100644 --- a/docs/osquery-manager.md +++ b/docs/osquery-manager.md @@ -1,9 +1,9 @@ # Osquery Manager -[SOC](security-onion-console.md) includes a link on the sidebar which takes you to the Osquery Manager page inside [Kibana](kibana.md). +[Security Onion Console](security-onion-console.md) includes a link on the sidebar which takes you to the Osquery Manager page inside [Kibana](kibana.md). ## More Information !!! NOTE - For more information about Osquery Manager, please see . \ No newline at end of file + For more information about Osquery Manager, please see . diff --git a/docs/passwords.md b/docs/passwords.md index a483eb62..9ce96fef 100644 --- a/docs/passwords.md +++ b/docs/passwords.md @@ -18,7 +18,7 @@ Your default user account should have sudo permissions. Command-line utilities t Log into [SOC](security-onion-console.md) using the username and password you created in the Setup wizard or the username and password provided by your administrator. -You can change your password in [SOC](security-onion-console.md) by clicking the user icon in the upper-right corner, clicking `Settings`, and then going to the `Security` tab. Please note that, due to technical limitations, if you change your SOC password here it will not update your password in [influxdb](influxdb.md). However, resetting your password via [Administration](administration.md) will reset your [influxdb](influxdb.md) password. +You can change your password in [SOC](security-onion-console.md) by clicking the user icon in the upper-right corner, clicking `Settings`, and then going to the `Security` tab. Please note that, due to technical limitations, if you change your SOC password here it will not update your password in [InfluxDB](influxdb.md). However, resetting your password via [Administration](administration.md) will reset your [InfluxDB](influxdb.md) password. If you've forgotten your SOC password, an administrator can change it using the [Administration](administration.md) interface. diff --git a/docs/pcap.md b/docs/pcap.md index 4a1113d3..361b4174 100644 --- a/docs/pcap.md +++ b/docs/pcap.md @@ -1,6 +1,6 @@ # PCAP -[SOC](security-onion-console.md) includes a PCAP interface which allows you to access your full packet capture that was written to disk by [Suricata](suricata.md). +[Security Onion Console](security-onion-console.md) includes a PCAP interface which allows you to access your full packet capture that was written to disk by [Suricata](suricata.md). You can access PCAP in two different ways. The first and most common option is to pivot to PCAP from a particular event in [Alerts](alerts.md), [Dashboards](dashboards.md), or [Hunt](hunt.md) by choosing the PCAP action on the action menu. @@ -37,5 +37,5 @@ If you have trouble retrieving PCAP, here are some things to check: - Verify that full packet capture is enabled via [Suricata](suricata.md). - Check to see if you have any [BPF](bpf.md) configuration that may cause [Suricata](suricata.md) to ignore the traffic. - Check [Grid](grid.md) and verify that all services are running properly. -- Check [influxdb](influxdb.md) and verify that PCAP Retention is long enough to include the stream you're looking for. -- Make sure that there is plenty of free space on `/nsm` to carve the stream and write the output to disk. \ No newline at end of file +- Check [InfluxDB](influxdb.md) and verify that PCAP Retention is long enough to include the stream you're looking for. +- Make sure that there is plenty of free space on `/nsm` to carve the stream and write the output to disk. diff --git a/docs/performance.md b/docs/performance.md index 2b10cf2f..ae8caca6 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -2,7 +2,7 @@ ## CPU Affinity/Pinning -For best performance, CPU intensive processes like [Zeek](zeek.md) and [Suricata](suricata.md) should be pinned to specific CPUs. In most cases, you'll want to pin sniffing processes to the same CPU that your sniffing NIC is bound to. For more information, please see the Performance subsection in the appropriate [Suricata](suricata.md) and [Zeek](zeek.md) sections. +For best performance, CPU intensive processes like [Zeek](zeek.md) and [Suricata](suricata.md) should be pinned to specific CPUs. In most cases, you'll want to pin sniffing processes to the same CPU that your sniffing NIC is bound to. For more information, please see the Performance subsection in the appropriate [Suricata](suricata.md) and [Zeek](zeek.md) sections. ## Misc @@ -12,17 +12,14 @@ Consider adopting some of the suggestions from here: - - -## RSS - -Check your sniffing interfaces to see if they have Receive Side Scaling (RSS) queues. If so, you may need to reduce to 1: - - ## Disk/Memory If you have plenty of RAM, disable swap altogether: + Use `hdparm` to gather drive statistics and alter settings, as described here: + `vm.dirty_ratio` is the maximum amount of system memory that can be filled with dirty pages before everything must get committed to disk. @@ -30,7 +27,8 @@ Use `hdparm` to gather drive statistics and alter settings, as described here: `vm.dirty_background_ratio` is the percentage of system memory that can be filled with "dirty" pages, or memory pages that still need to be written to disk -- before the pdflush/flush/kdmflush background processes kick in to write it to disk. More information: + ## Elastic -You will want to make sure that each part of the pipeline is operating at maximum efficiency. Depending on your configuration, this may include [Elastic Agent](elastic-agent.md), [Logstash](logstash.md), [Redis](redis.md), and [Elasticsearch](elasticsearch.md). \ No newline at end of file +You will want to make sure that each part of the pipeline is operating at maximum efficiency. Depending on your configuration, this may include [Elastic Agent](elastic-agent.md), [Logstash](logstash.md), [Redis](redis.md), and [Elasticsearch](elasticsearch.md). diff --git a/docs/pfsense.md b/docs/pfsense.md index f334787e..a1352abf 100644 --- a/docs/pfsense.md +++ b/docs/pfsense.md @@ -18,7 +18,7 @@ To use the simple parser, first go to [Administration](administration.md) --> Co ![Image](images/config-item-firewall.png) -Once there, select the `syslog` option, specify the IP address of the pfSense firewall, and click the checkmark to save. Then click the `SYNCHRONIZE Grid` button under the `Options` menu at the top of the page. +Once there, select the `syslog` option, specify the IP address of the pfSense firewall, and click the checkmark to save. Then click the `SYNCHRONIZE GRID` button under the `Options` menu at the top of the page. Next, configure your pfSense firewall to send `syslog` to the IP address of your Security Onion box. If you are using pfSense 2.6.0 or higher, make sure that `Log Message Format` is set to `BSD (RFC 3164, default)`. @@ -30,7 +30,7 @@ The second option for collecting pfSense firewall logs is using the Elastic Inte First, add the pfSense integration and configure the pfSense firewall: -1. Go to [Elastic Fleet](elastic-fleet.md), click the `Agent policies` tab, and then click the desired policy (for example `so-Grid-nodes_general`). +1. Go to [Elastic Fleet](elastic-fleet.md), click the `Agent policies` tab, and then click the desired policy (for example `so-grid-nodes_general`). 2. Click the `Add integration` button. 3. Search for `pfSense` and then click on the `pfSense` integration. 4. The Elastic Integration page will show instructions for configuring pfSense. Follow these instructions but please note that the Elastic Integration expects to receive pfSense logs on port 9001 by default. @@ -45,6 +45,6 @@ Next, allow the traffic from the pfSense firewall to port 9001. These instructio 3. On the left side, go to `firewall`, select `hostgroups`, and click the `customhostgroup0` group. On the right side, enter the IP address of the pfSense firewall and click the checkmark to save. 4. On the left side, go to `firewall`, select `portgroups`, select the `customportgroup0` group, and then click `udp`. On the right side, enter `9001` and click the checkmark to save. 5. On the left side, go to `firewall`, select `role`, and then select the node type that will receive the pfSense logs. Then drill into `chain` --> `INPUT` --> `hostgroups` --> `customhostgroup0` --> `portgroups`. On the right side, enter `customportgroup0` and click the checkmark to save. -6. If you would like to apply the rules immediately, click the `SYNCHRONIZE Grid` button under the `Options` menu at the top of the page. +6. If you would like to apply the rules immediately, click the `SYNCHRONIZE GRID` button under the `Options` menu at the top of the page. Once all configuration is complete, you should be able to go to [Dashboards](dashboards.md) and select the `Firewall - pfSense` dashboard to see your firewall logs. \ No newline at end of file diff --git a/docs/post-installation.md b/docs/post-installation.md index 789807f1..537ede16 100644 --- a/docs/post-installation.md +++ b/docs/post-installation.md @@ -24,7 +24,7 @@ You can check the [Grid](grid.md) page to see if all services are running correc Please note that new nodes start off showing a red Fault and may take a few minutes to fully initialize before they show a green OK. -You can also verify services are running from the command line with the [SO-Status](so-status.md) command: +You can also verify services are running from the command line with the [so-status](so-status.md) command: ``` sudo so-status @@ -32,11 +32,11 @@ sudo so-status ## SSH -You should be able to do most administration from [SOC](security-onion-console.md) but if you need access to the command line then we recommend using [SSH](ssh.md) rather than the [Console](console.md). +You should be able to do most administration from [SOC](security-onion-console.md) but if you need access to the command line then you can use use [SSH](ssh.md). ## Data -- Review the [Elasticsearch](elasticsearch.md) section to see if you need to change any of the default settings. In particular, if you have a multi-node deployment with one or more search nodes, we HIGHLY recommend configuring ILM to delete indices before Elasticsearch reaches its watermark setting and stops ingesting new data. +- Review the [Elasticsearch](elasticsearch.md) section to see if you need to change any of the default settings. In particular, review the Data Stream Lifecycle Management (DLM) and make sure that your retention periods and available storage leave sufficient free space before Elasticsearch reaches its watermark setting and stops ingesting new data. - Review the [Full Packet Capture](full-packet-capture.md) and [Suricata](suricata.md) sections to see if you need to change the PCAP retention settings. @@ -48,4 +48,4 @@ You should be able to do most administration from [SOC](security-onion-console.m - Full-time analysts may want to connect using a dedicated [Security Onion Desktop](security-onion-desktop.md). -- Any IDS/NSM system needs to be tuned for the network it’s monitoring. Please see the [Detections](detections.md) and [Rules](rules.md) sections. \ No newline at end of file +- Any IDS/NSM system needs to be tuned for the network it’s monitoring. Please see the [Detections](detections.md) and [Rules](rules.md) sections. diff --git a/docs/postgresql.md b/docs/postgresql.md new file mode 100644 index 00000000..7b071d23 --- /dev/null +++ b/docs/postgresql.md @@ -0,0 +1,181 @@ +# PostgreSQL + +From : + +> PostgreSQL is a powerful, open source object-relational database system with over 35 years of active development that has earned it a strong reputation for reliability, feature robustness, and performance. + +Security Onion uses PostgreSQL as its central data platform, running on manager nodes. It backs the [Onion AI](onion-ai.md) conversation store and receives [Telegraf](influxdb.md) metrics alongside [InfluxDB](influxdb.md). + +## Storage + +- Data: `/nsm/postgres/` +- Configuration: `/opt/so/conf/postgres/` +- Secrets (mounted 0600 into the container): `/opt/so/conf/postgres/secrets/` +- Logs: `/opt/so/log/postgres/` + +!!! warning + + Do not manually modify files in `/nsm/postgres/` as this could corrupt the database. + +## Authentication + +PostgreSQL uses a three-tier authentication model: + +- **`postgres` superuser** — administrative operations only (schema init, role management during upgrade). The password is auto-generated and delivered to the container via the file mounted at `/run/secrets/postgres_password` using `POSTGRES_PASSWORD_FILE`. +- **`so_postgres` application user** — used by the Security Onion platform (the SOC assistant store) for day-to-day database access. The password is auto-generated and delivered via `/run/secrets/so_postgres_pass` using `SO_POSTGRES_PASS_FILE`. +- **`so_telegraf_` per-minion users** — one login role per grid minion, all members of the shared `so_telegraf` group. Each minion only sees its own credential via its per-minion pillar file. + +All credentials are auto-generated and managed by Salt. Passwords are delivered to the container as mounted files rather than plaintext environment variables so they do not appear in `docker inspect` output. + +## Encryption + +All TCP connections to PostgreSQL are encrypted with TLS using a certificate signed by the Security Onion CA at `/etc/pki/postgres.crt` / `/etc/pki/postgres.key`. + +`pg_hba.conf` only permits: + +- `local` Unix-socket connections (used by the container's own admin scripts). +- `hostssl` TLS-wrapped TCP connections with SCRAM-SHA-256 authentication. + +Plain-text `host` TCP connections are rejected, so a client using `sslmode=disable` cannot establish a session even if it has valid credentials. + +## Configuration + +You can configure PostgreSQL by going to [Administration](administration.md) --> Configuration --> postgres. + +Commonly adjusted settings: + +| Setting | Default | Description | +|---------|---------|-------------| +| `max_connections` | `100` | Maximum concurrent PostgreSQL connections. | +| `shared_buffers` | `256MB` | Memory for the shared buffer cache. Raising this improves read cache hit rate at the cost of RAM. | +| `log_min_messages` | `warning` | Minimum severity written to the PostgreSQL log. | +| `telegraf.retention_days` | `14` | Days of Telegraf metrics retained in the `so_telegraf` database. Older partitions are dropped hourly by pg_partman. | + +Infrastructure settings (TLS paths, `hba_file`, `listen_addresses`, preloaded extensions, etc.) are marked **advanced** — they are Salt-managed and should not be changed in normal operation. + +### Telegraf output selector + +The backend(s) Telegraf writes metrics to is configured under [Administration](administration.md) --> Configuration --> telegraf --> **output**: + +| Value | Behavior | +|-------|----------| +| `INFLUXDB` | Telegraf writes only to [InfluxDB](influxdb.md). | +| `POSTGRES` | Telegraf writes only to PostgreSQL (`so_telegraf` database). | +| `BOTH` | Telegraf dual-writes to both (useful during migration validation). | + +## Firewall + +PostgreSQL is only reachable from the manager itself and — within the docker bridge — from container peers. Access is enforced by the Security Onion [firewall](firewall.md) via the `DOCKER-USER` iptables chain: `postgres` (5432) is granted only to the `manager`, `managerhype`, `managersearch`, `eval`, and `standalone` hostgroups. Sensors, heavynodes, search nodes, receivers, and other roles cannot reach the port. + +## Management Commands + +Security Onion provides command-line tools for managing PostgreSQL. These tools run `psql` inside the [Docker](docker.md) container so PostgreSQL client binaries do not need to be installed on the host. + +### Interactive Shell + +Open an interactive `psql` session as the superuser: + +```bash +sudo so-postgres-manage shell +``` + +### Execute SQL + +Run a SQL command directly: + +```bash +sudo so-postgres-manage sql "SELECT version();" +``` + +### Execute SQL File + +Run a SQL script: + +```bash +sudo so-postgres-manage sqlfile /path/to/script.sql +``` + +### List Databases + +```bash +sudo so-postgres-manage dblist +``` + +### List Database Roles + +```bash +sudo so-postgres-manage userlist +``` + +### Container Lifecycle + +```bash +sudo so-postgres-start +sudo so-postgres-stop +sudo so-postgres-restart +``` + +### Grid Metrics Snapshot + +`so-stats-show` prints the most recent CPU, memory, disk, and load metrics for each host reporting to the Telegraf PostgreSQL backend. Useful for verifying that Telegraf is successfully landing metrics before consuming them in a dashboard. + +```bash +sudo so-stats-show # all hosts +sudo so-stats-show # a single host +``` + +This tool requires `telegraf.output` to be `POSTGRES` or `BOTH`. + +## Telegraf Metrics + +When `telegraf.output` is `POSTGRES` or `BOTH`, each grid minion writes its metrics directly to a shared `so_telegraf` database inside PostgreSQL using its own per-minion login role. + +- Tables live in the `telegraf` schema and are partitioned daily by [`pg_partman`](https://github.com/pgpartman/pg_partman). +- Retention is driven by the `postgres.telegraf.retention_days` configuration value; `pg_partman` drops expired partitions hourly via [`pg_cron`](https://github.com/citusdata/pg_cron). +- Tags and fields are stored as `jsonb` so the fixed-column-per-table limit (1600) does not constrain high-cardinality inputs. +- Each TCP connection is TLS-verified against the Security Onion CA (`sslmode=verify-full`). + +Per-minion credentials are provisioned automatically when a minion's key is accepted — a Salt reactor creates the database role and writes the password into that minion's own pillar file. No admin intervention is required. + +## Extensions + +The following PostgreSQL extensions are installed and available: + +| Extension | Description | +|-----------|-------------| +| **pgvector** | Vector similarity search for AI/ML embeddings. | +| **pg_cron** | Job scheduling within PostgreSQL. Drives pg_partman maintenance. | +| **pg_partman** | Time-based automated table partitioning. Drives Telegraf metric retention. | +| **pgcrypto** | Cryptographic functions for column-level encryption. | + +## Backup + +PostgreSQL is automatically backed up daily at 00:05. The backup uses `pg_dumpall` to capture all databases and roles, compressed with gzip, and stored 0600-mode at `/nsm/backup/so-postgres-backup-YYYY_MM_DD.sql.gz` with 7-day retention. + +See the [Backup](backup.md) page for more. + +To manually create a backup: + +```bash +docker exec so-postgres pg_dumpall -U postgres | gzip > /nsm/backup/so-postgres-manual-backup.sql.gz +``` + +To restore from a backup: + +```bash +zcat /nsm/backup/so-postgres-backup-2026_04_21.sql.gz | docker exec -i so-postgres psql -U postgres +``` + +## Diagnostic Logging + +PostgreSQL server logs land at `/opt/so/log/postgres/`. For container-level issues (startup problems, OOM, crashes), also check the [Docker](docker.md) logs: + +```bash +sudo docker logs so-postgres +``` + +## More Information + +!!! note + + For more information about PostgreSQL, please see . diff --git a/docs/proxy.md b/docs/proxy.md index 70e04dee..79861731 100644 --- a/docs/proxy.md +++ b/docs/proxy.md @@ -4,7 +4,7 @@ Setup will ask if you want to connect through a proxy server and, if so, it will ![Image](images/18_setup_direct_proxy.png) -If you have problems installing via your proxy server, you may want to consider the [airgap](airgap.md) option as everything will install via the ISO image. +If you have problems installing via your proxy server, you may want to consider the [Airgap](airgap.md) option as everything will install via the ISO image. ## Configuration @@ -51,7 +51,11 @@ To configure git to use a proxy for all users, add the following to `/etc/gitcon If you're going to run something using sudo, remember to use the `-i` option to force it to process the environment variables. For example: - sudo -i so-rule-update + sudo -i so-suricata-restart !!! WARNING - Using `sudo su -` will ignore `/etc/environment`, instead use `sudo su` if you need to operate as root. \ No newline at end of file + Using `sudo su -` will ignore `/etc/environment`, instead use `sudo su` if you need to operate as root. + +## NIDS Rules + +If you are using a proxy and need to download NIDS rulesets, you will also need to configure proxy settings for the NIDS ruleset downloads. These settings are separate from the system-wide proxy configuration above. See the [NIDS](nids.md) documentation for details on configuring the Proxy URL, Proxy Username, Proxy Password, and Proxy CA Path for ruleset downloads. diff --git a/docs/rbac.md b/docs/rbac.md index 89ddcd66..920611d8 100644 --- a/docs/rbac.md +++ b/docs/rbac.md @@ -6,47 +6,62 @@ RBAC in Security Onion covers both Security Onion privileges and Elastic stack p ## Default Roles -Security Onion ships with the following user roles: `superuser`, `analyst`, `limited-analyst`, `auditor`, and `limited-auditor`. +Security Onion ships with the following user roles: `superuser`, `analyst`, `limited-analyst`, `auditor`, `limited-auditor`, `subgrid-auditor`, and `subgrid-superuser`. See the table below which explains the specific Security Onion privileges granted to each role. -| | superuser | analyst | limited-analyst | auditor | limited-auditor | -|---|:---:|:---:|:---:|:---:|:---:| -| View alerts | X | X | X | X | X | -| Acknowledge alerts | X | X | X | | | -| Escalate alerts and events | X | X | X | | | -| View detections | X | X | X | X | X | -| Modify (and Delete) detections | X | X | | | | -| View events in Hunt | X | X | X | X | X | -| View own PCAP jobs | X | X | X | O | O | -| View all PCAP jobs | X | X | | X | | -| Pivot to PCAP job from event | X | X | X | | | -| Request arbitrary PCAP jobs | X | X | | | | -| Delete own PCAP job | X | X | X | O | O | -| Delete any PCAP job | X | X | | | | -| View all nodes in Grid | X | X | X | X | X | -| View all users | X | X | | X | | -| View all users' roles | X | X | | X | | -| View own user | X | X | X | X | X | -| View own user roles | X | X | X | X | X | -| Change own password | X | X | X | X | X | -| Add, update, and reset SOC users | X | | | | | -| Modify and synchronize Grid config | X | | | | | -| Manage Grid membership of nodes | X | | | | | -| Initiate node actions (Ex: reboot) | X | | | | | -| Manage API clients | X | | | | | -| View and list existing API clients | X | | X | | -| Manage Active Queries | X | | | | | -| View Playbooks | X | X | X | X | X | -| Chat with Onion AI | X | X | | | | -| Delete Own Onion AI Sessions | X | X | | | | -| View Own Onion AI History | X | X | | | | -| View All Users' Onion AI History | X | | | | | +| | superuser | analyst | limited-analyst | auditor | limited-auditor | subgrid-superuser | subgrid-auditor | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| View alerts | X | X | X | X | X | | | +| Acknowledge alerts | X | X | X | | | | | +| Escalate alerts and events | X | X | X | | | | | +| View detections | X | X | X | X | X | | | +| Modify (and Delete) detections | X | X | | | | | | +| View events in Hunt | X | X | X | X | X | | | +| View own PCAP jobs | X | X | X | O | O | | | +| View all PCAP jobs | X | X | | X | | | | +| Pivot to PCAP job from event | X | X | X | | | | | +| Request arbitrary PCAP jobs | X | X | | | | | | +| Delete own PCAP job | X | X | X | O | O | | | +| Delete any PCAP job | X | X | | | | | | +| View all nodes in Grid | X | X | X | X | X | | | +| View all users | X | X | | X | | | | +| View all users' roles | X | X | | X | | | | +| View own user | X | X | X | X | X | | | +| View own user roles | X | X | X | X | X | | | +| Change own password | X | X | X | X | X | | | +| Add, update, and reset SOC users | X | | | | | | | +| Modify and synchronize Grid config | X | | | | | | | +| Manage Grid membership of nodes | X | | | | | | | +| Initiate node actions (Ex: reboot) | X | | | | | | | +| Manage API clients | X | | | | | | | +| View and list existing API clients | X | | X | | | | +| Manage Active Queries | X | | | | | | | +| View Playbooks | X | X | X | X | X | | | +| Chat with Onion AI | X | X | | | | | | +| Delete Own Onion AI Sessions | X | X | | | | | | +| View Own Onion AI History | X | X | | | | | | +| View All Users' Onion AI History | X | | | | | | | +| View own Onion AI memories | X | X | | X | | | | +| View global Onion AI memories | X | | | X | | | | +| View all users' Onion AI memories | X | | | X | | | | +| Manage own Onion AI memories | X | X | | | | | | +| Manage global Onion AI memories | X | | | | | | | +| Manage all users' Onion AI memories | X | | | | | | | +| View notifications | X | X | X | X | X | | | +| View all notifications | X | | | X | | | | +| Manage notifications | X | X | X | | | | | +| Read subgrid data | X | | | | | X | X | +| Modify subgrid data | X | | | | | X | | !!! NOTE Both `auditor` and `limited-auditor` roles can interact with previously created PCAPs if they were created before a user was converted to that role (e.g. user was downgraded from `analyst` to `auditor`). This is denoted by **O** in the above table. +!!! NOTE + + The `subgrid-auditor` and `subgrid-superuser` roles are used in [Manager of Managers](manager-of-managers.md) deployments to grant read-only or read/write access to remote subgrids. Users needing subgrid access should be assigned one of these roles in addition to their primary role (e.g., `analyst`). + !!! NOTE A system role called `agent` is used by the Security Onion agent that runs on each node of the Security Onion Grid. This special role is given the *jobs/process*, *nodes/read*, and *nodes/write* permissions (defined at the bottom of this page). Avoid creating custom roles that share the same name as Security Onion-provided roles. @@ -63,6 +78,16 @@ In the [Administration](administration.md) interface, navigate to the Users scre In the [Administration](administration.md) interface, navigate to the Users screen and click the > icon to the left of the email address needing adjusting. Check or uncheck the desired roles. + +## Default Role Assignment + +When a user is created, they can optionally be automatically assigned to a specific role. To enable this automatic assignment, locate the `defaultRole` configuration setting and specify the desired default role name. This automatic assignment will occur when the user logs into SOC for the first time. + +!!! WARNING + + If an administrator removes all role assignments from a user and the user logs back in that user will again be automatically assigned the default role. Always lock inactive users instead of removing role assignments from a user. + + ## Creating Custom Roles !!! WARNING @@ -161,7 +186,7 @@ These steps will guide you through an example where we wish to introduce a new r sudo so-elasticsearch-query _security/privilege/kibana-.Kibana | jq '. | map_values(keys)' ``` -3. Run so-checkin from the manager: +3. [Auto State Apply](salt.md#auto-state-apply) picks up the new role file within a few minutes. If you don't want to wait, run so-checkin from the manager: ``` @@ -216,7 +241,7 @@ The available low-level Security Onion privileges are listed in the table below: | *events/read* | Read from Elasticsearch | | *events/write* | Write to Elasticsearch | | *events/ack* | Acknowledge alerts | -| *Grid/read* | Read information about the Grid and its node memberships | +| *Grid/read* | Read information about the grid and its node memberships | | *Grid/write* | Accept and reject Grid memberships from new and existing nodes | | *jobs/read* | View all PCAP jobs | | *jobs/pivot* | Pivot to PCAP job from event | @@ -235,9 +260,21 @@ The available low-level Security Onion privileges are listed in the table below: | *playbooks/read* | View all playbooks | | *playbooks/write* | Currently unused | | *playbooks/delete* | Currently unused | +| *subgrid/read* | Read data from subgrids | +| *subgrid/write* | Modify data on subgrids | +| *notifications/read* | View notifications | +| *notifications/read_all* | View all notifications | +| *notifications/write* | Create and update notifications | +| *memory/read_authored* | View own Onion AI memories | +| *memory/read_global* | View global Onion AI memories | +| *memory/read_all* | View all Onion AI memories | +| *memory/write_self* | Create and update own Onion AI memories | +| *memory/write_global* | Create and update global Onion AI memories | +| *memory/write_all* | Create and update all Onion AI memories | | *assistant/read_authored* | View own Onion AI conversation history | | *assistant/write_authored* | Chat with Onion AI | | *assistant/delete_authored* | Delete own Onion AI conversation history | +| *assistant/read_shared* | View shared Onion AI conversation history | | *assistant/read_all* | View all Onion AI conversation history | | *assistant/write_all* | Currently unused | | *assistant/delete_all* | Currently unused | @@ -270,7 +307,17 @@ These discrete privileges are then collected into privilege groups as defined be | user-monitor | *roles/read*, *users/read* | | playbook-monitor | *playbooks/read* | | playbook-admin | *playbooks/read*, *playbooks/write*, *playbooks/delete* | -| assistant-user | *assistant/read_authored*, *assistant/write_authored*, *assistant/delete_authored* | -| assistant-admin | *assistant/read_authored*, *assistant/write_authored*, *assistant/delete_authored*, *assistant/read_all*, *assistant/write_all*, *assistant/delete_all* | +| subgrid-admin | *subgrid/read*, *subgrid/write* | +| subgrid-monitor | *subgrid/read* | +| notification-admin | *notifications/read*, *notifications/write* | +| notification-monitor | *notifications/read* | +| notification-auditor | *notifications/read*, *notifications/read_all* | +| memory-user | *memory/read_authored*, *memory/write_self* | +| memory-curator | *memory/read_authored*, *memory/read_global*, *memory/write_self*, *memory/write_global* | +| memory-admin | *memory/read_authored*, *memory/read_global*, *memory/read_all*, *memory/write_self*, *memory/write_global*, *memory/write_all* | +| memory-monitor | *memory/read_authored*, *memory/read_global*, *memory/read_all* | +| assistant-user | *assistant/read_authored*, *assistant/write_authored*, *assistant/delete_authored*, *assistant/read_shared* | +| assistant-admin | *assistant/read_authored*, *assistant/write_authored*, *assistant/delete_authored*, *assistant/read_shared*, *assistant/read_all*, *assistant/write_all*, *assistant/delete_all* | +| assistant-monitor | *assistant/read_authored*, *assistant/read_shared*, *assistant/read_all* | † intended for use by Sensoroni agents only \ No newline at end of file diff --git a/docs/release-notes.md b/docs/release-notes.md index a7d575c1..7b888369 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -2,7 +2,222 @@ ### Known Issues -For all known issues, please see . +#### Elastic Images and CPU Support + +Elastic recently changed made a change which requires x86_64-v3 CPU support: + + + +If you are running Security Onion in a Proxmox VM, we recommend setting your CPU to host to make sure that your x86_64-v3 CPU is passed through to the VM: + + +If your CPUs do not support x86_64-v3 at all, then we recommend holding off on this upgrade until Elastic has resolved this issue. ### Release History +3.3.0 Hotfix [20260911] Changes +------------------------------- + +- FIX: Disable memory by default #16232 + +3.3.0 [20260908] Changes +------------------------ + +- FEATURE: 508 AA - Contrast +- FEATURE: Add subgrid roles for MoM grids +- FEATURE: Advanced Agent Studio +- FEATURE: Agentic memory +- FEATURE: Allow for tuning logstash log.level and log.format #16183 +- FEATURE: Allow for tuning multiple Logstash pipelines in SOC #15090 +- FEATURE: ES Troubleshoot from Grid Screen #15432 +- FEATURE: Improve keyboard trap handling for 508 compliance +- FEATURE: Initial notification framework +- FEATURE: Onion AI Reports Tools +- FEATURE: Reduce reboot frequency by removing non-UEK kernels #16208 +- FEATURE: Support drag/drop for config list ordering, such as custom queries +- FIX: Avoid highlighting multiple menu entries #16129 +- FIX: Correct API /.well-known/* paths to properly resolve #16198 +- FIX: Hypervisor not running first highstate #14947 +- FIX: Improve HTTP authorization and security +- FIX: Improve PCAP parsing and viewing +- FIX: Improve upload size enforcement logic #16197 +- FIX: In Cases -> Events tab the Hunt icons link to ?q=[object+Object] instead of valid hunt query #16177 +- FIX: In Hunt interface the Basic Metrics section minimized state does not persist across page refreshes #16154 +- FIX: Local state files not triggering auto state apply #16136 +- FIX: Logrotate for hypervisor +- FIX: Onion AI Reports Font Key Parsing Error +- FIX: Remove usage of scripted fields +- FIX: Re-running so-setup on airgap installs fails +- FIX: so-boot-highstate.service is never enabled on non-manager nodes #16166 +- FIX: SOC Alerts interface in 3.x field and values not selectable, only values are selectable #16155 +- FIX: Soup error due to template DBs collation mismatch #16138 +- FIX: Soup force remote nodes to highstate #16137 +- FIX: Switch telegraf command vars to array #16020 +- FIX: Zeek LogExpireInterval value available in SOC but requires zeekctl cron to be run manually #16156 +- UPGRADE: CyberChef to 11.3.0 #16128 +- UPGRADE: Elasticsearch 9.4.5 +- UPGRADE: PrismJS to 1.30 #16134 +- UPGRADE: Salt-bootstrap +- UPGRADE: SOC Go dependencies #16186 +- UPGRADE: Strelka Go components to latest commit #16141 +- UPGRADE: Telegraf to 1.39.3 #16196 +- UPGRADE: Zeek to 8.0.10 #16171 + +3.2.0 [20260729] Changes +---------------------- + +- FEATURE: Initial implementation of agentic framework +- FEATURE: Autodetect salt apply state from SOC config audit history +- FEATURE: Config audit history w/ restore +- FEATURE: Datastream Lifecycle Management +- FEATURE: Guided Analysis Progressive Loading #16091 +- FEATURE: Limit certain config settings to specific node types #15972 +- FEATURE: Map Antivirus Sigma rules to Elastic Defend #14468 +- FEATURE: Sigma Playbooks - Initial Set #16090 +- FEATURE: Support ES|QL in Sigma detections +- FEATURE: Support Suricata Transactional rule dir #15948 +- FEATURE: Updated default Hunt query #16026 +- FIX: Allow manager to run two full highstates during soup #15986 +- FIX: Allow periods in NIC names #16060 +- FIX: Disable Zeek icsnpp-modbus script #16110 +- FIX: Do not allow login redirects to API URLs #16065 +- FIX: Elastic Defend incompatible with linux 7+ kernels +- FIX: Elastic Fleet server state persistence #16051 +- FIX: Elasticfleet: server urls auto updating when opted out #15960 +- FIX: Elasticsearch GC log rotate #16034 +- FIX: Elasticsearch: index template partial duplicate #15959 +- FIX: Ensure so-yaml.py updated during soup #16066 +- FIX: Import/Eval error in elasticsearch configuration script #16025 +- FIX: Improve elastic agent install outcome to check that the installation is healthy +- FIX: Improve Elasticsearch scripts runtime #15987 +- FIX: Improve Group Metrics Layout on Alert Page +- FIX: Improve Hunt Query Box UI on Smaller Screens +- FIX: Improve logging when Alerts fail to Ack #15999 +- FIX: Missing esheap pillar value breaks highstate on Elasticsearch nodes #16108 +- FIX: Nav Bar Hover Misalignment +- FIX: Refreshing browser while in hunt drops index filters #15331 +- FIX: Rename Connect API to Security Onion API #15921 +- Fix: Rework soup postupgrade_changes #15946 +- FIX: Run Elastic Agent regenerate installers script +- FIX: Salt: server restart and highstate issues +- FIX: so-start | so-stop | so-restart utilities #16014 +- FIX: Soup should run so-config-backup script #15901 +- FIX: Soup verify an upgrade is available prior to running elasticsearch upgrade compatibility check +- FIX: Suricata rule reload should not report failure if a reload is already in progress #16016 +- UPGRADE: alpine base images to 3.24.1 #15992 +- UPGRADE: Axios to 1.18.1 #15950 +- UPGRADE: CyberChef to 11.2.0 #15997 +- UPGRADE: Dompurify to 3.4.12 #15977 +- UPGRADE: Elasticsearch 9.3.7 #16063 +- UPGRADE: golang to 1.26.4 #15988 +- UPGRADE: InfluxDB to 2.9.1 (UI to 2.9.0) #15993 +- UPGRADE: js-yaml to 4.3.0 #15976 +- UPGRADE: Kafka to 4.3.1 #16001 +- UPGRADE: Kratos and Hydra google/x/net Go deps #16048 +- UPGRADE: Migration of more images to UBI 9.7 #16010 +- UPGRADE: nginx to 1.31.2 #15996 +- UPGRADE: node to 26.3.1 #15990 +- UPGRADE: OpenCanary to 0.9.8 #16003 +- UPGRADE: Oracle UEK8 Kernel +- UPGRADE: Postgres to 17.10 #15998 +- UPGRADE: pySigma & sigma-cli #15616 +- UPGRADE: Redis to 7.4.9 #15994 +- UPGRADE: registry to 3.1.1 #15991 +- UPGRADE: SOC Go dependencies #16032 +- UPGRADE: Suricata to 8.0.6 #16042 +- UPGRADE: Telegraf to 1.39.0 #15995 +- UPGRADE: Zeek to 8.0.9 #16040 + +3.1.0 Hotfix [20260528] Changes +------------------------------- + +- FIX: Grids with multiple heavy nodes fail Elasticsearch upgrade verification for 3.1.0 +- FIX: Grids using custom logstash pipeline(s) may have stale pillar entries #15932 + +3.1.0 [20260521] Changes +------------------------ + +- FEATURE: Add Postgres support for future features +- FEATURE: Add bonded NIC support for management interfaces #15548 +- FEATURE: Add ingest latency metric +- FEATURE: Allow the setup of bond1 for management for ISO installs #15865 +- FEATURE: Elastic Fleet continuously validate output policy +- FEATURE: RAID monitoring for hypervisor VMs #15809 +- FEATURE: Restore Suricata Overrides from backup #15881 +- FEATURE: Sigma mappings - M365 & Fortigate #15882 +- FEATURE: Simplified Onion AI setup for regions outside US #15773 +- FEATURE: Support Azure OpenAI endpoints #15841 +- FIX: 'Investigate' using inaccessible local model shows "insufficient credits" +- FIX: Add options selection to annotations #15744 +- FIX: Appliance images in SOC grid misaligned #15713 +- FIX: Consider setting Elastic Agent output level to warning only #15431 +- FIX: Deterministically sort threshold.conf #15815 +- FIX: Improve elastic agent install outcome to check that the installation is healthy +- FIX: Improve lucene and elastic query param validation #15860 +- FIX: Improve reverse DNS lookups success rate #15760 +- FIX: Improve usability for visually impaired users +- FIX: JA4+ license hyperlink #15717 +- FIX: Make SOC and Kratos enabled annoations readonly #15827 +- FIX: Modifying detection templates in config causes SOC to crash loop #15798 +- FIX: Need better user feedback when attaching assistant chat to a case #15689 +- FIX: Node descriptions containing both spaces and numbers prevent pillar creation #15540 +- FIX: Prevent excessive OnionAI query length +- FIX: Reactor sominion_setup #15834 +- FIX: Refactor Detections backup #14992 +- FIX: Reinstall #15811 +- FIX: SOUP verify all Elasticsearch nodes are compatible with the next Elasticsearch version #15908 +- FIX: Suricata pcap-log max-files rounds to 0 when calculated value is between 0 and 1 #15740 +- FIX: UI should show the name of the current Dashboard #15703 +- FIX: Use hunt action link for case observable hunt pivots #15752 +- FIX: Use safeload for loading filecheck config #15859 +- FIX: Zeek ingest pipeline for JA4d.log #15886 +- UPGRADE: Axios to 1.15.0 in SOC #15774 +- UPGRADE: CyberChef to 11.0.0 #15890 +- UPGRADE: Elasticsearch to 9.3.3 +- UPGRADE: Kratos and Hydra 26.2.0+pgx #15796 +- UPGRADE: SOC Go dependencies #15795 +- UPGRADE: SOC frontend dependency libs #15848 +- UPGRADE: Suricata to 8.0.5 #15903 +- UPGRADE: Zeek to 8.0.8 #15794 +- UPGRADE: nginx to 1.30.1 #15891 + +3.0.0 [20260331] Changes +---------------------- + +- FEATURE: Configurable Elasticsearch vm.max_map_count setting +- FEATURE: Dynamically load Zeek plugins on zeek startup #15546 +- FEATURE: Enable JA4+ License Acceptance #15560 +- FEATURE: Parsing for Zeek websockets logs #15657 +- FEATURE: Refresh login page with updated look +- FEATURE: Refresh SOC UI with updated look +- FEATURE: Support additional alt names in web cert +- FEATURE: Support docker ulimit customization #15581 +- FEATURE: Suricata PCAP replacing Stenographer +- FIX: API 401 errors will no longer redirect #15611 +- FIX: Cleanup file.absent and cron.absent +- FIX: Detections - Intermittent "error closing scroll" #14216 +- FIX: Duplicated user roles when refreshing frontend at Administration > Users #15688 +- FIX: Enabled / Disabled Buttons for SOC Grid Configuration Options #15649 +- FIX: Fix rule validators in SOC #15533 +- FIX: Global override configs should not apply to certain indices #15601 +- FIX: Network Transport for suricata alerts should be lowercase #15668 +- FIX: Sensors are not checking in while processing long jobs #15650 +- FIX: so-suricata-testrule script #15396 +- FIX: STIG V1R3 +- FIX: Suricata address-groups vars allow negation #15664 +- FIX: Unable to create detections via Security Onion API #15673 +- UPGRADE: All frontend 3rd party deps +- UPGRADE: ATTACK Navigator to 5.3.0 #15680 +- UPGRADE: CyberChef to 10.22.1 #15681 +- UPGRADE: ElastAlert2 to 2.28.0 #15685 +- UPGRADE: Golang 3rd party deps #15647 +- UPGRADE: Golang to 1.26.1 #15580 +- UPGRADE: Hydra to 25.4.0 #15678 +- UPGRADE: Kafka to 3.9.2 #15684 +- UPGRADE: Kratos to 25.4.0 #15677 +- UPGRADE: Nginx to 1.29.6 #15686 +- UPGRADE: OpenCanary to 0.9.7 #15679 +- UPGRADE: Redis to 7.2.13 #15682 +- UPGRADE: Suricata to 8.0.4 #15625 +- UPGRADE: Telegraf to 1.38.0 #15683 +- UPGRADE: Update Docker base images diff --git a/docs/removing-a-node.md b/docs/removing-a-node.md index 29cf424b..e11ed15c 100644 --- a/docs/removing-a-node.md +++ b/docs/removing-a-node.md @@ -24,11 +24,11 @@ Once all data has migrated off the search node, then you can continue with other ## Removing from Salt -You can remove a node from [salt](salt.md) by going to [Administration](administration.md) --> Grid Members. +You can remove a node from [Salt](salt.md) by going to [Administration](administration.md) --> Grid Members. ![Image](images/84_gridmembers.png) -Find the Grid Member you would like to remove, click the `REVIEW` button, and then click the `DELETE` button. +Find the grid Member you would like to remove, click the `REVIEW` button, and then click the `DELETE` button. ## Removing from SOC diff --git a/docs/reports.md b/docs/reports.md index 8fb56336..6c681151 100644 --- a/docs/reports.md +++ b/docs/reports.md @@ -1,6 +1,6 @@ # Reports -Starting in version 2.4.180, Security Onion Pro added support for producing reports and data exports for analytical purposes. This is not a data migration or full data export feature but rather is intended for analysts to present relevant data to the broader team. Consequently, exports are limited in the amount of data that will be produced. +Security Onion Pro customers can produce reports and data exports for analytical purposes. This is not a data migration or full data export feature but rather is intended for analysts to present relevant data to the broader team. Consequently, exports are limited in the amount of data that will be produced. !!! NOTE @@ -43,7 +43,7 @@ Case reports can be generated by opening a case and clicking the Export icon abo Customer feedback will help the Security Onion team determine which additional reports will be provided in future releases. Provide feedback by reaching out to our Support team. -Standard reports can be lightly customized via the Configuration screen. Search for the report name, such as "Case report". Customizations include changing layout formatting, text content, and hiding or showing additional data. The ability to show additional data is limited to what the built-in report has already queried. For example, the built-in Case Report shows the TLP of each observable, but not the PAP. So if you want to see the PAP and not the TLP, then you could customize to do so. Remember to synchronize the Grid after saving customizations. +Standard reports can be lightly customized via the Configuration screen. Search for the report name, such as "Case report". Customizations include changing layout formatting, text content, and hiding or showing additional data. The ability to show additional data is limited to what the built-in report has already queried. For example, the built-in Case Report shows the TLP of each observable, but not the PAP. So if you want to see the PAP and not the TLP, then you could customize to do so. Remember to synchronize the grid after saving customizations. ### Custom Reports @@ -117,7 +117,7 @@ The second tabular output is showing the list of events (up to 100 as previously Customizing reports can be intimidating to those not familiar with this level of detail. Pro customers with professional service hours can utilize the experience of the Security Onion support team to help get you started. -Remember to synchronize the Grid after saving custom reports. +Remember to synchronize the grid after saving custom reports. ## Tabular Exports @@ -131,7 +131,7 @@ Also included with this feature is the ability to export data from several of th The exported data will be saved to a CSV file and include the headers as the first line of the CSV. The CSV will be generated in the background and will then be available for download in the Reports interface. -Group and Event data can be exported by clicking on the data export icon, typically found in the top-left corner of the Group metric table or graph, or the event table. While the export icon is visible on the graph mode of a group metric, it will still export the underlying data in CSV format. It does not attempt to export the graph visualization. However, starting with this same version 2.4.180, users can now use the browser's Print feature to send those graph visualizations and dashboards to a printer or PDF file. +Group and Event data can be exported by clicking on the data export icon, typically found in the top-left corner of the Group metric table or graph, or the event table. While the export icon is visible on the graph mode of a group metric, it will still export the underlying data in CSV format. It does not attempt to export the graph visualization. However, users can use the browser's Print feature to send those graph visualizations and dashboards to a printer or PDF file. CSV exports will export all underlying data, up to a configured max. Therefore, while the [Dashboards](dashboards.md) interface could show 10 entries in a Group metric because of how the group limits are set, the CSV could contain more than the 10, if the backing data has more aggregated metrics available. @@ -280,4 +280,4 @@ If there had been a second `groupby` clause then it would have been referenced a ``` {{ .SomeField | join "," }} -``` \ No newline at end of file +``` diff --git a/docs/rules.md b/docs/rules.md index 15f1883b..b7bfba13 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -1,9 +1,3 @@ -# Rules +# Rules Overview Security Onion supports three main types of rules: [NIDS](nids.md), [Sigma](sigma.md), and [YARA](yara.md). You can manage all three types via [Detections](detections.md). - -## Table of Contents - -- [NIDS](nids.md) -- [Sigma](sigma.md) -- [YARA](yara.md) \ No newline at end of file diff --git a/docs/salt.md b/docs/salt.md index 5b8a46a2..78289a77 100644 --- a/docs/salt.md +++ b/docs/salt.md @@ -39,6 +39,49 @@ If you want to force a node to do a full update of all salt states, you can run sudo so-checkin ``` +## Auto State Apply + +When you save a configuration change in [Administration](administration.md) --> Configuration, or when rules are updated on the manager node, Security Onion detects the change and applies the affected state to only the nodes that need it. This typically happens within a few minutes instead of waiting for the next scheduled highstate. + +Files that you create or edit by hand on the manager in any of the following directories are detected the same way: + +| Directory | What it holds | +|-----------|---------------| +| `/opt/so/saltstack/local/salt/zeek/policy/` | [Zeek](zeek.md) intel and custom policies | +| `/opt/so/saltstack/local/salt/zeek/zkg/` | [Zeek](zeek.md) custom packages | +| `/opt/so/saltstack/local/salt/elasticsearch/files/ingest/` | [Elasticsearch](elasticsearch.md) custom ingest parsers | +| `/opt/so/saltstack/local/salt/elasticsearch/roles/` | [RBAC](rbac.md) custom Elastic stack role files | +| `/opt/so/saltstack/local/salt/logstash/pipelines/config/custom/` | [Logstash](logstash.md) custom pipeline configuration files | + +Other files under `/opt/so/saltstack/local/salt/` are not watched and are picked up at the next scheduled highstate (see [Highstate Interval](#highstate-interval)). + +Auto State Apply is configured at [Administration](administration.md) --> Configuration --> salt --> auto_apply: + +| Setting | Default | Description | +|---------|---------|-------------| +| `enabled` | `true` | Enables or disables Auto State Apply. When disabled, changes are picked up at the next scheduled highstate. | +| `debounce_seconds` | `30` | How long a change must be quiet before it is applied. Multiple changes made within this window are combined into a single update. | +| `drain_interval` | `15` | How often, in seconds, the manager checks for changes that are ready to be applied. | +| `batch` | `25%` | How many nodes apply the state at once, either a number such as `10` or a percentage such as `25%`. | +| `batch_wait` | `15` | How many seconds to wait between each batch of nodes. | + +Other than `enabled`, these are advanced settings, so you will only see them if you click the `Options` menu at the top of the page and then enable the `Show advanced settings` option. + +## Highstate Interval + +Every node also runs a scheduled highstate as a backstop. The interval is controlled by the `highstate_interval_minutes` setting at [Administration](administration.md) --> Configuration --> salt --> schedule and defaults to `120` minutes. The minimum value is `15` minutes. This is an advanced setting, so you will need to enable the `Show advanced settings` option to see it. + +## Reverting to Previous Behavior + +If you would like the grid to behave like previous versions of Security Onion, go to [Administration](administration.md) --> Configuration --> salt and: + +- set `auto_apply` --> `enabled` to `false` +- set `schedule` --> `highstate_interval_minutes` to `15` + +!!! WARNING + + If you disable Auto State Apply but leave the highstate interval at the default of `120` minutes, it can take up to two hours for a change to reach the rest of the grid. + ## Configuration Many of the options that are configurable in Security Onion are done by going to [Administration](administration.md) and then Configuration. @@ -62,4 +105,4 @@ The root cause of this error is a state trying to run on a minion when another s !!! NOTE - For more information about Salt, please see . \ No newline at end of file + For more information about Salt, please see . diff --git a/docs/security-onion-app-for-splunk.md b/docs/security-onion-app-for-splunk.md index f39a72d1..29c5679b 100644 --- a/docs/security-onion-app-for-splunk.md +++ b/docs/security-onion-app-for-splunk.md @@ -10,7 +10,7 @@ Enterprise customers utilizing Splunk can now install the Security Onion App for !!! NOTE - The Security Onion App for Splunk utilizes the Security Onion Connect API, which is an enterprise-level feature of Security Onion. Contact Security Onion Solutions, LLC via our website at for more information about purchasing a Security Onion Pro license to enable this feature. + The Security Onion App for Splunk utilizes the Security Onion API, which is an enterprise-level feature of Security Onion. Contact Security Onion Solutions, LLC via our website at for more information about purchasing a Security Onion Pro license to enable this feature. ## Configuration @@ -18,4 +18,4 @@ See to get started. !!! NOTE - A Connect API Client must be created in the Security Onion API Clients screen. The API Client should be granted sufficient permissions needed to perform the tasks that the Splunk app will need to execute. \ No newline at end of file + A Security Onion API Client must be created in the API Clients screen. The API Client should be granted sufficient permissions needed to perform the tasks that the Splunk app will need to execute. \ No newline at end of file diff --git a/docs/security-onion-console-customization.md b/docs/security-onion-console-customization.md index 887c77d5..33bd3f51 100644 --- a/docs/security-onion-console-customization.md +++ b/docs/security-onion-console-customization.md @@ -4,7 +4,7 @@ You can customize [Security Onion Console](security-onion-console.md) by going t ![Image](images/config-item-soc.png) -Below are some ways in which you can customize SOC. Once all customizations are complete, you can make the changes take effect by clicking the `Options` bar at the top and then clicking the `SYNCHRONIZE Grid` button. +Below are some ways in which you can customize SOC. Once all customizations are complete, you can make the changes take effect by clicking the `Options` bar at the top and then clicking the `SYNCHRONIZE GRID` button. ![Image](images/88_config_options.png) @@ -14,12 +14,7 @@ You can customize the SOC login page with a login banner by going to [Administra ## Overview Page -After logging into SOC, you'll start on the main SOC Overview page which can be customized as well. You can customize this by going to [Administration](administration.md) --> Configuration --> SOC --> files --> SOC --> Overview Page. This uses Markdown format as mentioned above. - -You can add images but they must be hosted from another host that is accessible by the user's browser. For example, let's use one of the images from our online documentation. The markdown to add that image would look like this: - - -![SOC Dashboards](https://docs.securityonion.net/en/2.4/_images/53_dashboards.png) +After logging into SOC, you'll start on the main SOC Overview page which can be customized as well. You can customize this by going to [Administration](administration.md) --> Configuration --> SOC --> files --> SOC --> Overview Page. This uses Markdown format as mentioned above. You can add images but they must be hosted from another host that is accessible by the user's browser. ## Links @@ -81,4 +76,4 @@ Please note that some events may not have GeoIP information and this query would - `SOC` - Enables the built-in Case Management, with our Escalation menu (default). -- `elasticcases` - Enables escalation to the [Elastic Cases](https://www.elastic.co/guide/en/security/current/cases-overview.html) tool. Escalations will always open a new case; there will not be an advanced escalation menu popup. This module will use the same user/pass that SOC uses to talk to Elastic. Note, however, that Elastic cases is actually a Kibana feature, therefore, when this setting is used, SOC will be communicating with the local Kibana service (via its API) for case escalations. \ No newline at end of file +- `elasticcases` - Enables escalation to the [Elastic Cases](https://www.elastic.co/guide/en/security/current/cases-overview.html) tool. Escalations will always open a new case; there will not be an advanced escalation menu popup. This module will use the same user/pass that SOC uses to talk to Elastic. Note, however, that Elastic cases is actually a Kibana feature, therefore, when this setting is used, SOC will be communicating with the local Kibana service (via its API) for case escalations. diff --git a/docs/security-onion-console.md b/docs/security-onion-console.md index b075a186..0155c909 100644 --- a/docs/security-onion-console.md +++ b/docs/security-onion-console.md @@ -1,8 +1,8 @@ -# Security Onion Console +# Security Onion Console Overview ![Image](images/diagrams/analyst.png) -Once all configuration is complete, you can then connect to Security Onion Console with your web browser. We recommend chromium-based browsers such as Google Chrome. Other browsers may work, but fully updated chromium-based browsers provide the best compatibility. +Once all configuration is complete, you can then connect to Security Onion Console (SOC) with your web browser. We recommend chromium-based browsers such as Google Chrome. Other browsers may work, but fully updated chromium-based browsers provide the best compatibility. Depending on the options you chose in the installer, connect to the IP address or hostname of your Security Onion installation. Then login using the email address and password that you specified in the installer. @@ -15,21 +15,3 @@ Once logged in, you'll notice the user menu in the upper-right corner. This allo On the left side of the page, you'll see links for analyst tools like [Alerts](alerts.md), [Dashboards](dashboards.md), [Hunt](hunt.md), [Cases](cases.md), [Detections](detections.md), [PCAP](pcap.md), [Kibana](kibana.md), [CyberChef](cyberchef.md), and [Attack Navigator](attack-navigator.md). While [Alerts](alerts.md), [Dashboards](dashboards.md), [Hunt](hunt.md), [Cases](cases.md), [Detections](detections.md), and [PCAP](pcap.md) are built into SOC itself, the remaining tools are external and will spawn separate browser tabs. If you'd like to customize SOC, please see the [SOC Customization](security-onion-console-customization.md) section. If you'd like to learn more about SOC logs, please see the [SOC Logs](security-onion-console-logs.md) section. - -## Table of Contents - -- [Alerts](alerts.md) -- [Dashboards](dashboards.md) -- [Hunt](hunt.md) -- [Cases](cases.md) -- [Detections](detections.md) -- [PCAP](pcap.md) -- [Grid](grid.md) -- [Downloads](downloads.md) -- [Administration](administration.md) -- [Kibana](kibana.md) -- [Elastic Fleet](elastic-fleet.md) -- [Osquery Manager](osquery-manager.md) -- [InfluxDB](influxdb.md) -- [CyberChef](cyberchef.md) -- [Attack Navigator](attack-navigator.md) \ No newline at end of file diff --git a/docs/security-onion-desktop.md b/docs/security-onion-desktop.md index 73cb909c..04fbaffc 100644 --- a/docs/security-onion-desktop.md +++ b/docs/security-onion-desktop.md @@ -1,4 +1,4 @@ -# Security Onion Desktop +# Security Onion Desktop Overview Full-time analysts may want to use a dedicated Security Onion desktop. This allows you to investigate pcaps, malware, and other potentially malicious artifacts without impacting your Security Onion deployment or your usual desktop environment. @@ -20,19 +20,19 @@ There are a few different ways to install Security Onion Desktop: !!! NOTE - Depending on how you install, it may take a full [salt](salt.md) cycle before all desktop components are installed and ready for use. + Depending on how you install, it may take a full [Salt](salt.md) cycle before all desktop components are installed and ready for use. **Joining to Grid** -You can optionally join your Desktop installation to your Grid. This allows it to pull updates from the Grid and automatically trust the Grid's HTTPS certificate. It also updates the manager's firewall to allow the Desktop installation to connect. Starting with Security Onion 2.4.20, Desktop nodes will now display on the [Grid](grid.md) page along with the other Grid nodes. +You can optionally join your Desktop installation to your grid. This allows it to pull updates from the grid and automatically trust the grid's HTTPS certificate. It also updates the manager's firewall to allow the Desktop installation to connect. Desktop nodes display on the [Grid](grid.md) page along with the other Grid nodes. -If you choose not to join your Desktop installation to your Grid, then you may need to allow the traffic through the host-based [firewall](firewall.md) by going to [Administration](administration.md) --> Configuration --> firewall --> hostgroups --> analyst. +If you choose not to join your Desktop installation to your grid, then you may need to allow the traffic through the host-based [firewall](firewall.md) by going to [Administration](administration.md) --> Configuration --> firewall --> hostgroups --> analyst. ![Image](images/config-item-firewall.png) **Disabling** -The analyst desktop is controlled via [salt](salt.md) pillar. If you need to disable the Security Onion Desktop environment, find the `workstation` setting in your [salt](salt.md) pillar and change `enabled: true` to `enabled: false`: +The analyst desktop is controlled via [Salt](salt.md) pillar. If you need to disable the Security Onion Desktop environment, find the `workstation` setting in your [Salt](salt.md) pillar and change `enabled: true` to `enabled: false`: ```yaml @@ -40,9 +40,3 @@ workstation: gui: enabled: false ``` - -## Table of Contents - -- [Chromium](chromium.md) -- [NetworkMiner](networkminer.md) -- [Wireshark](wireshark.md) \ No newline at end of file diff --git a/docs/security-onion-pro.md b/docs/security-onion-pro.md index 14a7935c..3c890d4c 100644 --- a/docs/security-onion-pro.md +++ b/docs/security-onion-pro.md @@ -1,24 +1,5 @@ -# Security Onion Pro - - +# Security Onion Pro Overview !!! NOTE Contact Security Onion Solutions, LLC via our website at for more information about purchasing a Security Onion Pro license to enable these features. - -## Table of Contents - -- [OIDC](oidc.md) -- [LUKS](luks.md) -- [FIPS](fips.md) -- [STIG](stig.md) -- [Notifications](notifications.md) -- [Guaranteed Message Delivery](kafka.md) -- [Connect API](connect-api.md) -- [Active Query Management](active-query-management.md) -- [Manager of Managers (MoM)](manager-of-managers.md) -- [MCP Server](mcp-server.md) -- [Security Onion App for Splunk](security-onion-app-for-splunk.md) -- [Hypervisor](hypervisor.md) -- [Reports](reports.md) -- [Onion AI](onion-ai.md) diff --git a/docs/sigma-logsource-coverage.md b/docs/sigma-logsource-coverage.md new file mode 100644 index 00000000..f8e46891 --- /dev/null +++ b/docs/sigma-logsource-coverage.md @@ -0,0 +1,104 @@ +# Sigma Logsource Coverage + +[Sigma](sigma.md) rules declare the telemetry they need via a `logsource` (product + category or service). Whether a rule can actually fire on your grid depends on two things: whether Security Onion **collects** that telemetry, and whether the Sigma **conversion pipeline** maps the rule's logsource and field names onto the fields Security Onion actually stores. This page maps the endpoint and cloud logsources in the SigmaHQ ruleset across both, so you can tell at a glance which rules work out of the box, which need additional endpoint configuration, and which need telemetry Security Onion does not collect. + +By default Security Onion imports the SigmaHQ **core** package (stable/test rules of high/critical level) plus the **emerging threats add-on** — see [Sigma Packages](sigma.md#sigma-packages) to change this. Detection counts are shown as **default (all rules)**: rules in the default packages, with the **all-rules** package total (every stable/test/experimental rule of medium level and above) in parentheses. Counts are current as of mid-2026. Note that a rule being *supported* is separate from it being *enabled* — see [Enable Sigma Rules on Import](sigma.md#enable-sigma-rules-on-import) for controlling which rules are active by default. + +**How to read the tables:** + +- ✅ **Supported** — telemetry is collected once the Security Onion [Elastic Agent](elastic-agent.md) (which includes Elastic Defend) is deployed to the endpoint, and the conversion pipeline scopes the rule correctly. No extra configuration. +- ⚠️ **Partial** — a subset of the category's telemetry is collected or mapped; some rules in the category will never match. See notes. +- 🔧 **Requires setup** — telemetry exists only if you configure something extra (deploy [Sysmon](sysmon.md), enable a Windows audit feature, or add an [integration](third-party-integrations.md)). +- ❌ **Not collected** — no equivalent telemetry; these rules will never fire. + +The **Playbooks** column shows Guided Analysis Playbook status for each logsource (see [Detections](detections.md)): ✅ playbooks are published for this logsource, 🔄 in development, **Planned**, or — not currently planned. Where the status is the same for every logsource in a table, it is stated in the section text instead of a column. + +## Windows endpoint — Elastic Defend + +Elastic Defend is the default endpoint telemetry agent for Security Onion. These categories are fully covered by Defend events, and the conversion pipeline scopes each one to the matching Defend event type (shown in the Telemetry column). They work out of the box. Guided Analysis Playbooks are published for all of these categories. + +| Logsource | Detections | Telemetry | Notes | +|---|---:|---|---| +| process_creation | 732 (1,297) | ✅ `endpoint.events.process` | | +| file_event | 130 (199) | ✅ `endpoint.events.file` | | +| registry_set | 113 (208) | ✅ `endpoint.events.registry` | Registry value modifications | +| image_load | 56 (109) | ✅ `endpoint.events.library` | Includes DLL hashes and code signatures | +| registry_event | 30 (40) | ⚠️ Partial | Value modifications only — see below | +| network_connection | 25 (51) | ✅ `endpoint.events.network` | `DestinationHostname` matches may not fire — Defend rarely records destination domains | +| dns_query | 10 (24) | ✅ `endpoint.events.network` (DNS) | Includes the querying process | +| driver_load | 7 (9) | ✅ `endpoint.events.library` | Scoped to `event.category:driver` | +| file_delete | 4 (12) | ✅ `endpoint.events.file` | | +| file_rename | 0 (1) | ✅ `endpoint.events.file` | | + +⚠️ **registry_event** is partial: Defend records registry *value modifications* only. Rules matching key paths work, but rules requiring key create/delete/rename semantics need [Sysmon](sysmon.md) EID 12/14 (7 of the 40 all-rules detections). Playbooks cover the Defend-answerable subset. + +## Windows endpoint — requires Sysmon + +Elastic Defend does not emit equivalent events for these categories. Deploying [Sysmon](sysmon.md) with an appropriate configuration provides the telemetry; without it, these rules will never fire. Guided Analysis Playbooks are not currently planned for these categories. + +!!! WARNING + + The conversion pipeline does not currently add Sysmon channel or event-ID scoping for these categories — converted rules rely on their field names alone to select the right events. Because some Sysmon fields are renamed to generic ECS fields (e.g. `SourceImage` → `process.executable`), a converted rule can match unrelated events. If you deploy Sysmon and enable rules in these categories, review the converted queries (Detection Source → Convert) and expect to tune. + +| Logsource | Detections | Sysmon telemetry | Notes | +|---|---:|---|---| +| process_access | 18 (24) | EID 10 | Cross-process handle access (e.g. LSASS dumping) — Defend's API telemetry does not record it | +| pipe_created | 11 (18) | EID 17/18 | | +| create_remote_thread | 9 (12) | EID 8 | | +| create_stream_hash | 6 (9) | EID 15 | | +| registry_delete | 3 (10) | EID 12 | | +| registry_add | 2 (3) | EID 12 | | +| sysmon_status, sysmon_error | 2 (2) | Service events | Only meaningful if Sysmon is deployed | +| file_change | 1 (1) | EID 2 | File creation-time change (timestomping) | +| process_tampering | 0 (1) | EID 25 | | +| file_executable_detected | 0 (1) | EID 29 | | +| file_access | 0 (6) | ❌ Not reliably collected | Needs ETW kernel-file auditing; neither Defend nor Sysmon provides general file-access events | + +## Windows event log channels + +These logsources target specific Windows event log channels. The default agent policy collects the core channels (Security, System, Application, Windows Defender, PowerShell, WMI-Activity); a few sources additionally require the corresponding Windows feature or audit policy to be enabled on the endpoint. + +| Logsource | Detections | Channel | Playbooks | Notes | +|---|---:|---|---|---| +| security | 92 (141) | ✅ Security | Planned | Some rules need specific audit policies enabled (e.g. object access, audit-log-cleared) | +| ps_script | 57 (146) | 🔧 Script block logging | 🔄 In development | Channel collected by default; enable script block logging on the endpoint via GPO | +| system | 46 (69) | ✅ System | Planned | | +| ps_module | 18 (28) | 🔧 Module logging | Planned | Channel collected by default; enable module logging via GPO | +| application | 19 (27) | ✅ Application | Planned | | +| windefend | 13 (15) | ✅ Defender Operational | Planned | | +| ps_classic_start, ps_classic_provider_start, powershell-classic | 4 (9) | ✅ PowerShell (classic) | 🔄 In development | Collected by default | +| wmi_event, wmi | 2 (5) | ✅ WMI-Activity | ✅ | Collected by default — see below | +| other service channels | ~41 (~76) | 🔧 Varies | — | ~30 low-volume channels — see below | + +✅ **wmi_event, wmi** — the WMI-Activity Operational channel natively records WMI provider loads and permanent event subscriptions (EID 5857–5861), so Sysmon EID 19–21 is not required. However, rules matching on the Sysmon field names (`EventNamespace`, `Consumer`, `Filter`) will not fire against the channel's fields; match on the event ID instead. + +🔧 **Other service channels** — ~30 low-volume channels (AppLocker, BITS, CodeIntegrity, Exchange management, task scheduler, print service, NTLM, …). Conversion scoping is automatic, but these channels are not collected by the default agent policy; each needs its channel added and, in some cases, the Windows feature enabled. + +## Linux and macOS endpoints + +Elastic Defend covers the core endpoint categories on Linux and macOS as well. + +| Logsource | Detections | Telemetry | Playbooks | Notes | +|---|---:|---|---|---| +| linux / process_creation | 51 (110) | ✅ Elastic Defend | Planned | | +| linux / auditd | 12 (28) | 🔧 auditd + integration | — | Needs auditd on the endpoint plus the Auditd integration; rule field names (`type`, `a0`, …) also need custom field mapping | +| linux / file_event | 10 (14) | ✅ Elastic Defend | Planned | | +| linux / network_connection | 5 (5) | ✅ Elastic Defend | Planned | | +| linux / other | ~20 (~26) | 🔧 Syslog / integrations | — | sshd, sudo, cron, clamav, vsftpd, guacamole, generic syslog. Only `linux/auth` has a conversion mapping; the rest need custom field mappings | +| macos / process_creation | 10 (49) | ✅ Elastic Defend | Planned | | +| macos / file_event | 2 (3) | ✅ Elastic Defend | Planned | | + +## Cloud and SaaS + +These require the corresponding Elastic [integration](third-party-integrations.md) to be configured — nothing is collected out of the box. Conversion support varies: **m365 rules are fully mapped** to the O365 integration's fields and dataset; the other products currently have no conversion mappings, so even with the integration collecting data, their rules need custom field mappings before they can match. Guided Analysis Playbooks are not currently planned for these logsources. + +| Logsource | Detections | Required integration | Conversion mapping | +|---|---:|---|---| +| azure (activitylogs, auditlogs, signinlogs, riskdetection, pim) | 51 (123) | Azure | — | +| aws / cloudtrail | 12 (42) | AWS CloudTrail | — | +| okta | 6 (20) | Okta | — | +| bitbucket / audit | 4 (12) | Bitbucket audit logs | — | +| github / audit | 4 (9) | GitHub audit log | — | +| m365 (threat_management, audit, exchange) | 1 (18) | Microsoft 365 | ✅ `o365.audit` | +| gcp (gcp.audit, google_workspace) | 0 (25) | GCP / Google Workspace | — | +| kubernetes / audit | 0 (9) | Kubernetes audit | — | diff --git a/docs/sigma.md b/docs/sigma.md index aae2a53c..dabc46ce 100644 --- a/docs/sigma.md +++ b/docs/sigma.md @@ -47,7 +47,7 @@ To add a new Sigma rule, go to the main [Detections](detections.md) page and cli 1. Click the Language drop-down and select `Sigma`. 2. Optionally specify a license. 3. Add the signature. -4. Click the `CREATE` button and the detection should deploy to your Grid at the next 15-minute cycle. +4. Click the `CREATE` button and the detection should deploy to your grid at the next 15-minute cycle. ![Image](images/59_detection_create.png) diff --git a/docs/so-detections-overrides-import.md b/docs/so-detections-overrides-import.md new file mode 100644 index 00000000..c0a477ce --- /dev/null +++ b/docs/so-detections-overrides-import.md @@ -0,0 +1,49 @@ +# so-detections-overrides-import + +`so-detections-overrides-import` imports detection overrides (the modify, suppress, and threshold tuning that is created from the TUNING tab in [Detections](detections.md)) back into the `so-detection` index. It is intended for restoring overrides captured by the nightly `so-detections-backup` script, for example when migrating to or rebuilding a Manager. + +It reads `.` files from a source directory (one override per line, NDJSON), finds the matching detection by `publicId` and engine, validates each override against the same rules SOC enforces, dedupes against overrides that already exist on the detection, and appends any that are new. + +!!! WARNING + + The nightly backup (`so-detections-backup`) does not delete a backup file when you remove an override in SOC, so the source directory may contain overrides you intentionally deleted. Review the source directory and remove any unwanted files before importing. + +## Usage + +Run on the manager and point `--source` at the directory containing your backup files: + +``` +sudo so-detections-overrides-import --source /nsm/backup/detections/repo/suricata/overrides/ +``` + +Currently only the `suricata` (NIDS) engine is supported. The matching backup files use the `.txt` extension. + +### Dry Run + +Preview exactly what would be added, skipped, or rejected without writing anything to Elasticsearch: + +``` +sudo so-detections-overrides-import --source /path/to/overrides --dry-run +``` + +## Options + +| Option | Description | +| --- | --- | +| `--source`, `-s` | **(Required)** Source directory containing `.` override files. | +| `--engine`, `-e` | Detection engine. Default: `suricata`. | +| `--dry-run`, `-n` | Print what would happen without writing to Elasticsearch. | +| `--no-import-note` | Do not prepend `[Imported YYYY-MM-DD] ` to each imported override's note. | +| `--index`, `-i` | Elasticsearch index to update. Default: `so-detection`. | + +## Behavior Notes + +- **Validation** mirrors the checks SOC applies in the UI. Invalid overrides are reported (with file and line number) and skipped rather than imported. +- **Deduplication** compares only the operational fields of an override (for example, type, track, and IP for a suppression), so re-running an import does not create duplicates and does not depend on timestamps or enabled state. +- **Missing detections**: if no detection matches a file's `publicId` and engine, that file is skipped and counted under "no detection." +- **Import note**: unless `--no-import-note` is set, each imported override's note is prefixed with `[Imported YYYY-MM-DD] ` so imported tuning is easy to identify. +- **Custom Suricata variables**: if an imported override references a Suricata variable that is not one of the built-in variables, the tool lists it at the end. You must define any such variable in SOC Config (Suricata variables) before the affected rules will function correctly. + +## More Information + +For more information about tuning detections and managing NIDS rules, see the [Detections](detections.md) and [NIDS](nids.md) sections. diff --git a/docs/so-import-pcap.md b/docs/so-import-pcap.md index 6a1c7b55..42a68be6 100644 --- a/docs/so-import-pcap.md +++ b/docs/so-import-pcap.md @@ -1,20 +1,20 @@ # so-import-pcap -`so-import-pcap` will import one or more pcaps into Security Onion and preserve original timestamps. It will do the following: +`so-import-pcap` will import one or more pcap files into Security Onion and preserve original timestamps. It will do the following: - generate IDS alerts using [Suricata](suricata.md) - generate network metadata using [Zeek](zeek.md) - store IDS alerts and network metadata in [Elasticsearch](elasticsearch.md) with original timestamps -- store pcaps where [SOC](security-onion-console.md) can find them +- store pcap files where [SOC](security-onion-console.md) can find them - provide a hyperlink for you to view all alerts and logs in [SOC](security-onion-console.md) !!! TIP - You can run this command manually, but for most use cases it's easier to upload a PCAP via [Grid](grid.md) and it will automatically run `so-import-pcap` for you. + You can run this command manually, but for most use cases it's easier to upload a pcap file via [Grid](grid.md) and it will automatically run `so-import-pcap` for you. ## Configuration -so-import-pcap requires you to run through Setup and choose a configuration that supports so-import-pcap. This includes Import Node and other nodes that include sensor services like Eval and Standalone. The quickest and easiest option is to choose Import Node which gives you the minimal services necessary to import a PCAP. +so-import-pcap requires you to run through Setup and choose a configuration that supports so-import-pcap. This includes Import Node and other nodes that include sensor services like Eval and Standalone. The quickest and easiest option is to choose Import Node which gives you the minimal services necessary to import a pcap file. !!! WARNING @@ -22,24 +22,24 @@ so-import-pcap requires you to run through Setup and choose a configuration that ## Usage -Once Setup completes, you can then run `sudo so-import-pcap` and supply the full path to at least one PCAP file. For example, to import a single PCAP named `import.PCAP`: +Once Setup completes, you can then run `sudo so-import-pcap` and supply the full path to at least one pcap file. For example, to import a single pcap file named `import.pcap`: ``` -sudo so-import-pcap /full/path/to/import.PCAP +sudo so-import-pcap /full/path/to/import.pcap ``` -To import multiple pcaps: +To import multiple pcap files: ``` -sudo so-import-pcap /full/path/to/import1.PCAP /full/path/to/import2.PCAP +sudo so-import-pcap /full/path/to/import1.pcap /full/path/to/import2.pcap ``` -Please note that if you import multiple pcaps at one time, so-import-pcap currently only provides a hyperlink for the last PCAP in the list. If you need a hyperlink for each PCAP, then you can run one PCAP file per so-import-pcap and use a for-loop to iterate over your collection of PCAP files. +Please note that if you import multiple pcap files at one time, so-import-pcap currently only provides a hyperlink for the last pcap file in the list. If you need a hyperlink for each pcap file, then you can run one pcap file per so-import-pcap and use a for-loop to iterate over your collection of pcap files. -so-import-pcap calculates the MD5 hash of the imported PCAP and creates a directory in `/nsm/import/` for that hash. This is where so-import-pcap stores the alerts and logs generated by the traffic in the PCAP. If you try to import that same PCAP again, it will tell you that it has already imported that PCAP. If for some reason you really do need to import that PCAP again, you can remove that PCAP's directory in `/nsm/import/` and then try again. +so-import-pcap calculates the MD5 hash of the imported pcap file and creates a directory in `/nsm/import/` for that hash. This is where so-import-pcap stores the alerts and logs generated by the traffic in the pcap file. If you try to import that same pcap file again, it will tell you that it has already imported that pcap file. If for some reason you really do need to import that pcap file again, you can remove that pcap's directory in `/nsm/import/` and then try again. ## Examples -If you don't already have some PCAP files to import, see [PCAPs](pcaps.md) for a list of sites where you can download sample pcaps. +If you don't already have some pcap files to import, see [PCAPs](pcaps.md) for a list of sites where you can download sample pcap files. -Our Quick Malware Analysis series at uses so-import-pcap to import pcaps from and other sites. Following along with these blog posts in your own so-import-pcap VM is a great way to practice your skills! \ No newline at end of file +Our Quick Malware Analysis series at uses so-import-pcap to import pcap files from and other sites. Following along with these blog posts in your own so-import-pcap VM is a great way to practice your skills! diff --git a/docs/so-user.md b/docs/so-user.md index cf49c9e4..7d2ce758 100644 --- a/docs/so-user.md +++ b/docs/so-user.md @@ -1,6 +1,6 @@ # so-user -[SOC](security-onion-console.md) user management should normally be done via [Administration](administration.md) as shown in the [accounts](accounts.md) section. However, if for some reason you can't log into SOC, you can use `so-user` from the command line to manage SOC user accounts. +[SOC](security-onion-console.md) user management should normally be done via [Administration](administration.md) as shown in the [Accounts](accounts.md) section. However, if for some reason you can't log into SOC, you can use `so-user` from the command line to manage SOC user accounts. `so-user` has many different operations. You can see them all by running `so-user` with no options: @@ -24,4 +24,4 @@ If you've forgotten your password, you can reset it using the `password` operati sudo so-user password --email onionuser@example.com ``` -Once you've reset your password, you should be able to log into SOC and go back to managing user accounts via [Administration](administration.md) as shown in the [accounts](accounts.md) section. \ No newline at end of file +Once you've reset your password, you should be able to log into SOC and go back to managing user accounts via [Administration](administration.md) as shown in the [Accounts](accounts.md) section. diff --git a/docs/software-bill-of-materials.md b/docs/software-bill-of-materials.md new file mode 100644 index 00000000..1a44a490 --- /dev/null +++ b/docs/software-bill-of-materials.md @@ -0,0 +1,40 @@ +--- +hide: + - toc +--- + +The following table lists the major software projects integrated into the current version of Security Onion. + +| Product | Version | Author | Project URL | License | Description | +| :---- | ----- | :---- | :---- | :---- | :---- | +| Alpine Linux | 3.24.1 | Alpine Linux Development Team | [https://alpinelinux.org/](https://alpinelinux.org/) | GNU GPL Version 3 | Alpine Linux is a security-oriented, lightweight Linux distribution based on musl libc and busybox. | +| ATT&CK Navigator | 5.3.0 | MITRE | [https://github.com/mitre-attack/attack-navigator](https://github.com/mitre-attack/attack-navigator) | Apache License 2 | The ATT&CK Navigator is designed to provide basic navigation and annotation of ATT&CK matrices, something that people are already doing today in tools like Excel. | +| CyberChef | 11.3.0 | GCHQ | [https://github.com/gchq/CyberChef](https://github.com/gchq/CyberChef) | Apache License 2 | The "Cyber Swiss Army Knife" \- a web app for encryption, encoding, compression and data analysis. | +| Docker | 29.2.1 | Docker | [https://github.com/docker](https://github.com/docker) | Apache License 2 | Docker is a set of platform as a service products that use OS-level virtualization to deliver software in packages called containers. | +| ElastAlert | 2.31.0 | Jason Ertel | [https://github.com/jertel/elastalert2](https://github.com/jertel/elastalert2) | Apache License 2 | ElastAlert is a simple framework for alerting on anomalies, spikes, or other patterns of interest from data in Elasticsearch. | +| Elastic Agent | 9.4.5 | Elastic | [https://github.com/elastic/elastic-agent](https://github.com/elastic/elastic-agent) | Elastic License Version 2 | Beats is a free and open platform for single-purpose data shippers. They send data from hundreds or thousands of machines and systems to Logstash or Elasticsearch. | +| Elasticsearch | 9.4.5 | Elastic | [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch) | Elastic License Version 2 | Elasticsearch is a distributed, open source search and analytics engine for all types of data, including textual, numerical, geospatial, structured, and unstructured. | +| evtx | 0.8.9 | Omer Benamram | [https://github.com/omerbenamram/evtx](https://github.com/omerbenamram/evtx) | MIT License | A cross-platform parser for the Windows XML EventLog format. | +| evtx2es | 1.5.5 | Shinta Nakano | [https://github.com/Security-Onion-Solutions/evtx2es](https://github.com/Security-Onion-Solutions/evtx2es) | MIT License | A fast library for parsing and importing Windows Event Logs into Elasticsearch. | +| ExifTool | 12.60 | Phil Harvey | [https://github.com/exiftool/exiftool](https://github.com/exiftool/exiftool) | GNU GPL Version 3 | ExifTool is a platform-independent Perl library plus a command-line application for reading, writing and editing meta information in a wide variety of files. | +| Hydra | 26.2.0 | Ory | [https://github.com/ory/hydra](https://github.com/ory/hydra) | Apache License 2 | Ory Hydra is a hardened, OpenID Certified OAuth 2.0 Server and OpenID Connect Provider optimized for low-latency, high throughput, and low resource consumption | +| InfluxDB | 2.9.1 | InfluxData | [https://github.com/influxdata/influxdb/tree/main-2.x](https://github.com/influxdata/influxdb/tree/main-2.x) | MIT License | InfluxDB is an open source time series platform. This includes APIs for storing and querying data, processing it in the background for ETL or monitoring and alerting purposes, user dashboards, and visualizing and exploring the data and more. | +| Kibana | 9.4.5 | Elastic | [https://github.com/elastic/kibana](https://github.com/elastic/kibana) | Elastic License Version 2 | Kibana is an open source frontend application that sits on top of the Elastic Stack, providing search and data visualization capabilities for data indexed in Elasticsearch. | +| Kratos | 26.2.0 | Ory | [https://github.com/ory/kratos](https://github.com/ory/kratos) | Apache License 2 | Ory Kratos is the developer-friendly, security-hardened and battle-tested Identity, User Management and Authentication system for the Cloud. | +| Logstash | 9.4.5 | Elastic | [https://github.com/elastic/logstash](https://github.com/elastic/logstash) | Elastic License Version 2 | Logstash is a server-side data processing pipeline that ingests data from a multitude of sources simultaneously, transforms it, and then sends it to your favorite "stash." | +| NetworkMiner | 2.8.1 | Netresec | [https://www.netresec.com/?page=NetworkMiner](https://www.netresec.com/?page=NetworkMiner) | GNU GPL Version 2 | NetworkMiner is an open source Network Forensic Analysis Tool (NFAT) and can be used as a passive network sniffer/packet capturing tool in order to detect operating systems, sessions, hostnames, open ports etc. without putting any traffic on the network. NetworkMiner can also parse PCAP files for off-line analysis and to regenerate/reassemble transmitted files and certificates from PCAP files. Only included in Security Onion Desktop.| +| Nginx | 1.31.2 | NGINX | [https://github.com/nginxinc/docker-nginx](https://github.com/nginxinc/docker-nginx) | 2-Clause BSD License | Nginx is a web server that can also be used as a reverse proxy, load balancer, mail proxy and HTTP cache. | +| OpenCanary | 0.9.8 | Thinkst Canary | [https://github.com/thinkst/opencanary](https://github.com/thinkst/opencanary) | 3-Clause BSD License | OpenCanary is a multi-protocol network honeypot. It's primary use-case is to catch hackers after they've breached non-public networks. | +| Oracle Linux 9 | 9.7 | Oracle Linux | [https://www.oracle.com/linux/](https://www.oracle.com/linux/) | GNU GPL Version 2 | Oracle Linux is a Linux distribution packaged and freely distributed by Oracle, available partially under the GNU General Public License since late 2006\. It is compiled from Red Hat Enterprise Linux source code, replacing Red Hat branding with Oracle's. | +| pcapfix | 1.1.7 | Robert Krause | [https://github.com/Rup0rt/pcapfix/](https://github.com/Rup0rt/pcapfix/) | GNU GPL Version 3 | Pcapfix is a tool to repair your damaged or corrupted pcap and pcapng files. | +| PostgreSQL | 17.10 | The PostgreSQL Global Development Group | [https://postgresql.org](https://postgresql.org) | PostgreSQL License | PostgreSQL is a powerful, open source object-relational database system that uses and extends the SQL language combined with many features that safely store and scale the most complicated data workloads. | +| Python | 3.14 | Python Software Foundation | [https://github.com/python/](https://github.com/python/) | Python Software Foundation License Version 2 | Python is a programming language that lets you work quickly and integrate systems more effectively. | +| Redis | 7.4.9 | Redis | [https://github.com/redis/redis](https://github.com/redis/redis) | 3-Clause BSD License | Redis is an open source in-memory data structure store used as a database, cache and message broker. | +| Salt | 3006.19 | Salt Project | [https://github.com/saltstack/salt](https://github.com/saltstack/salt) | Apache License 2 | Salt is a distributed remote execution system used to execute commands and query data. It was developed in order to bring the best solutions found in the world of remote execution together and make them better, faster and more malleable. Salt accomplishes this via its ability to handle larger loads of information, and not just dozens, but hundreds or even thousands of individual servers, handle them quickly and through a simple and manageable interface. | +| Security Onion Console | 3.3.0 | Security Onion Solutions | [https://github.com/Security-Onion-Solutions/securityonion-soc](https://github.com/Security-Onion-Solutions/securityonion-soc) | Elastic License Version 2 | Security Onion Console is a web interface to viewing alerts, hunting, and PCAP analysis. | +| Strelka | 1.0.1 | Target | [https://github.com/target/strelka](https://github.com/target/strelka) | Apache License 2 | Strelka is a real-time file scanning system used for threat hunting, threat detection, and incident response. Strelka’s purpose is to perform file extraction and metadata collection at huge scale. | +| Suricata | 8.0.6 | OISF | [https://github.com/OISF/suricata](https://github.com/OISF/suricata) | GNU GPL Version 2 | Suricata is a free and open source, mature, fast and robust network threat detection engine. Suricata inspects the network traffic using a powerful and extensive rules and signature language. | +| Telegraf | 1.39.3 | InfluxData | [https://github.com/influxdata/telegraf](https://github.com/influxdata/telegraf) | MIT License | Telegraf is a server-based agent for collecting and sending all metrics and events from databases, systems, and IoT sensors. Telegraf is written in Go and compiles into a single binary with no external dependencies, and requires a very minimal memory footprint. | +| Yara | 4.5.4 | VirusTotal | [https://github.com/virustotal/yara](https://github.com/virustotal/yara) | 3-Clause BSD License | YARA is a tool aimed at (but not limited to) helping malware researchers to identify and classify malware samples. | +| Wireshark | 3.4.10 | Wireshark | [https://github.com/wireshark/wireshark](https://github.com/wireshark/wireshark) | GNU GPL Version 2 | Wireshark is the world’s foremost and widely-used network protocol analyzer. It lets you see what’s happening on your network at a microscopic level and is the de facto (and often de jure) standard across many commercial and non-profit enterprises, government agencies, and educational institutions. Only included in Security Onion Desktop. | +| Zeek | 8.0.10 | Zeek | [https://github.com/zeek/zeek/](https://github.com/zeek/zeek/) | 3-Clause BSD License | A powerful framework for network traffic analysis and security monitoring. | diff --git a/docs/soup.md b/docs/soup.md index fd78cb45..789013c8 100644 --- a/docs/soup.md +++ b/docs/soup.md @@ -23,7 +23,7 @@ To update your Security Onion deployment, run the `soup` command with sudo: If necessary, `soup` will update itself and then ask you to run `soup` again. Once `soup` is fully updated, it will then check for other updates. This includes Security Onion version updates, Security Onion hotfixes, and operating system (OS) updates. -After running `soup` or rebooting a Security Onion node, it may take a few minutes for services to display an `OK` status on the [Grid](grid.md) screen. This may be due to the intial on-boot [salt](salt.md) highstate running. If services do not appear to be fully up and running within 15 minutes, try running the following command: +After running `soup` or rebooting a Security Onion node, it may take a few minutes for services to display an `OK` status on the [Grid](grid.md) screen. This may be due to the intial on-boot [Salt](salt.md) highstate running. If services do not appear to be fully up and running within 15 minutes, try running the following command: sudo so-checkin @@ -32,9 +32,9 @@ After running `soup` or rebooting a Security Onion node, it may take a few minut When we release a new version of Security Onion, we update the [Release Notes](release-notes.md) section and publish a blog post to . You'll want to review these for any relevant information about the individual updates. -If `soup` finds a full version update, then it will update the Security Onion version in `/etc/soversion`, all [salt](salt.md) code, and all [Docker](docker.md) images. +If `soup` finds a full version update, then it will update the Security Onion version in `/etc/soversion`, all [Salt](salt.md) code, and all [Docker](docker.md) images. -`soup` automatically keeps the previous version of [Docker](docker.md) images. These older unused [Docker](docker.md) images will be automatically removed at the next version update. If you need to remove these older [docker](docker.md) images immediately, first verify that the upgrade completed successfully and that everything is working properly. You could then remove the older images individually or all at once using a command like: +`soup` automatically keeps the previous version of [Docker](docker.md) images. These older unused [Docker](docker.md) images will be automatically removed at the next version update. If you need to remove these older [Docker](docker.md) images immediately, first verify that the upgrade completed successfully and that everything is working properly. You could then remove the older images individually or all at once using a command like: sudo docker system prune -a @@ -43,7 +43,7 @@ However, please note that this an aggressive option and you should exercise caut ## Security Onion Hotfixes -`soup` checks for Security Onion hotfixes. Hotfixes typically include updates to the [salt](salt.md) code and small configuration changes that do not warrant a full version update. This does not include Docker images since that would require a full version update. +`soup` checks for Security Onion hotfixes. Hotfixes typically include updates to the [Salt](salt.md) code and small configuration changes that do not warrant a full version update. This does not include Docker images since that would require a full version update. After applying a hotfix, you may notice that the Security Onion version in `/etc/soversion` stays the same. The application of the hotfix is tracked on the manager in the `/etc/sohotfix` file. @@ -67,29 +67,22 @@ If you would like to prevent certain packages from being upgraded automatically `soup` will check for local configurations in `/opt/so/saltstack/local/` that may cause issues and flag them with the message `Potentially breaking changes found in the following files`. Please examine the output of `soup` and review any local configurations for possible issues. -## Detections - -If you are upgrading from a version older than 2.4.70, `soup` will do the following to prepare for migration to [Detections](detections.md): - -- Playbook Plays will be backed up to `/nsm/backup/detections-migration/` and any active ElastAlert rules will be backed up and removed. -- Suricata tuning configurations will be backed to `/nsm/backup/detections-migration/` and any thresholds will be migrated over to [Detections](detections.md). - ## Log If `soup` displays any errors, you can check `/root/soup.log` for additional clues. ## Airgap -To update an [airgap](airgap.md) deployment, you'll need to get the latest ISO image to the airgapped manager and then run `soup` which will ask where to find it: +To update an [Airgap](airgap.md) deployment, you'll need to get the latest ISO image to the airgapped manager and then run `soup` which will ask where to find it: - burn the latest ISO image to a DVD and insert it in the DVD drive of the manager (example: `/dev/cdrom`) - flash the ISO image to a USB drive and connect that USB drive to the manager (example: `/dev/sdb`) -- simply copy the ISO file itself to the manager (example: `/home/YourUser/securityonion-2.4.XYZ-YYYYMMDD.iso`) +- simply copy the ISO file itself to the manager (example: `/home/YourUser/securityonion-3.X.Y-YYYYMMDD.iso`) Instead of waiting for soup to prompt for the location, you can also specify the path on the command line using the `-f` option. For example (change this to reflect the actual path to the ISO file or disk device containing the ISO media): - sudo soup -y -f /home/YourUser/securityonion-2.4.XYZ-YYYYMMDD.iso + sudo soup -y -f /home/YourUser/securityonion-3.X.Y-YYYYMMDD.iso ## Elastic @@ -125,16 +118,16 @@ This will make `soup` proceed unattended, automatically answering `yes` to any p ### Data failed to compile -Occasionally, `soup` may output a `Data failed to compile` error that says something like `Rendering SLS failed: Jinja variable 'None' has no attribute`. In most cases, this error corrects itself on the next [salt](salt.md) run. +Occasionally, `soup` may output a `Data failed to compile` error that says something like `Rendering SLS failed: Jinja variable 'None' has no attribute`. In most cases, this error corrects itself on the next [Salt](salt.md) run. ### Pillars and sls files -`soup` will check [salt](salt.md) pillars to make sure they can be rendered. If not, it will output a message like this: +`soup` will check [Salt](salt.md) pillars to make sure they can be rendered. If not, it will output a message like this: There is an issue rendering the manager's pillars. Please correct the issues in the sls files mentioned below before running SOUP again. -This usually means that somebody has modified the [salt](salt.md) sls files and introduced a typo. +This usually means that somebody has modified the [Salt](salt.md) sls files and introduced a typo. ### Downloading images @@ -161,18 +154,18 @@ Here are some other errors that you may see when running `soup`: and/or - There is a problem downloading the so-xyz:2.4.0 image. Details: + There is a problem downloading the so-xyz:3.X.Y image. Details: gpg: Signature made Thu 18 Feb 2021 02:26:10 PM UTC using RSA key ID FE507013 gpg: BAD signature from "Security Onion Solutions, LLC " If you see these errors, it most likely means that a salt highstate process was already running when `soup` began. You can wait a few minutes and then try `soup` again. Alternatively, you can run `sudo so-checkin` and wait for it to complete before running `soup` again. ## Distributed deployments -If you have a distributed deployment with a manager node and separate sensor nodes and/or search nodes, you **only** need to run `soup` on the manager. Once `soup` has completed, other nodes should update themselves at the next [salt](salt.md) highstate (typically within 15 minutes). +If you have a distributed deployment with a manager node and separate sensor nodes and/or search nodes, you **only** need to run `soup` on the manager. Once `soup` has completed, other nodes should update themselves at the next [Salt](salt.md) highstate (typically within 15 minutes). !!! WARNING - Just because the update completed on the manager does NOT mean the upgrade is complete on other nodes in the Grid. Do not manually restart anything until you know that all the search nodes and heavy nodes are updated. + Just because the update completed on the manager does NOT mean the upgrade is complete on other nodes in the grid. Do not manually restart anything until you know that all the search nodes and heavy nodes are updated. Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, [Elasticsearch](elasticsearch.md) might not be able to talk to said heavy node until the update is complete. @@ -181,17 +174,17 @@ If you have a distributed deployment with a manager node and separate sensor nod When you run `soup` on the manager, it does the following: - Checks to see if it is running on a manager. -- Checks to see if the Grid is in [airgap](airgap.md) mode. If so, it will then ask for the location of the ISO or mount point. +- Checks to see if the grid is in [Airgap](airgap.md) mode. If so, it will then ask for the location of the ISO or mount point. - Checks to see if we're running the latest version of `soup`. If not, it will put the latest in the correct place and ask you to re-run `soup`. - Compares the installed version with what is available on github or the ISO image. -- Checks to see if [salt](salt.md) needs to be updated (more on this later). +- Checks to see if [Salt](salt.md) needs to be updated (more on this later). - Downloads the new [Docker](docker.md) images or, if airgap, copies them from the ISO image. -- Stops the [salt](salt.md) master and minion and restarts it in a restricted mode. This mode only allows the manager to connect to it so that we make sure the manager is done before any of the minions are updated. -- Updates [salt](salt.md) if necessary. This will cause the master and minion services to restart but still in restricted mode. +- Stops the [Salt](salt.md) master and minion and restarts it in a restricted mode. This mode only allows the manager to connect to it so that we make sure the manager is done before any of the minions are updated. +- Updates [Salt](salt.md) if necessary. This will cause the master and minion services to restart but still in restricted mode. - Makes any changes to pillars that are needed such as adding new settings or renaming values. This varies from release to release. -- If the Grid is in [airgap](airgap.md) mode, then it copies the latest ET Open rules and yara rules to the manager. -- The new [salt](salt.md) code is put into place on the manager. -- Runs a highstate on the manager which is the actual upgrade where it will use the new [salt](salt.md) code and [Docker](docker.md) containers. -- Unlocks the [salt](salt.md) master service and allows minions to connect again. -- Issues a command to all minions to update [salt](salt.md) if necessary. This is important to note as it takes time to to update the [salt](salt.md) minion on all minions. If the minion doesn't respond for whatever reason, it will not be upgraded at this time. This is not an issue because the first thing that gets checked when a minion talks to the master is if [salt](salt.md) needs to be updated and will apply the update if it does. +- If the grid is in [Airgap](airgap.md) mode, then it copies the latest ET Open rules and yara rules to the manager. +- The new [Salt](salt.md) code is put into place on the manager. +- Runs a highstate on the manager which is the actual upgrade where it will use the new [Salt](salt.md) code and [Docker](docker.md) containers. +- Unlocks the [Salt](salt.md) master service and allows minions to connect again. +- Issues a command to all minions to update [Salt](salt.md) if necessary. This is important to note as it takes time to to update the [Salt](salt.md) minion on all minions. If the minion doesn't respond for whatever reason, it will not be upgraded at this time. This is not an issue because the first thing that gets checked when a minion talks to the master is if [Salt](salt.md) needs to be updated and will apply the update if it does. - Nodes connect back to the manager and actually perform the upgrade to the new version. diff --git a/docs/ssh.md b/docs/ssh.md index 23e7e3a9..1df23d5d 100644 --- a/docs/ssh.md +++ b/docs/ssh.md @@ -1,3 +1,3 @@ # SSH -Security Onion uses the latest SSH packages. It does not manage the SSH configuration in `/etc/ssh/sshd_config` with [salt](salt.md). This allows you to add any PAM modules or enable two factor authentication (2FA) of your choosing. \ No newline at end of file +Security Onion uses the latest SSH packages. It does not manage the SSH configuration in `/etc/ssh/sshd_config` with [Salt](salt.md). This allows you to add any PAM modules or enable two factor authentication (2FA) of your choosing. \ No newline at end of file diff --git a/docs/stig.md b/docs/stig.md index b87ede1f..a3b6b112 100644 --- a/docs/stig.md +++ b/docs/stig.md @@ -28,17 +28,25 @@ In addition to the required partitions, using the STIG menu option will also con !!! WARNING Before enabling STIGs on your production Security Onion deployment, we recommend testing in a development environment. With different environments and configurations, there may be unexpected errors. + + Currently, not all node types fully support enabling STIGs. These include the following: + + - Hypervisor nodes + - VMs created within a hypervisor node do support enabling STIGs + - Heavy nodes + - IDH nodes -To enable STIGs you'll first need setup your Security Onion Grid and apply your [Security Onion Pro](security-onion-pro.md) license. You can then navigate to [Administration](administration.md) --> Configuration --> stig --> enabled and set the value to `true`. +To enable STIGs you'll first need to set up your Security Onion Grid and apply your [Security Onion Pro](security-onion-pro.md) license. You can then navigate to [Administration](administration.md) --> Configuration --> stig --> enabled and set the value to `true`. !!! NOTE You will need to enable the [Administration](administration.md) --> Show advanced settings option to modify this setting. ## OpenSCAP -In order to apply STIGs on Security Onion we use a combination of our existing Saltstack configuration managment and OpenSCAP. Currently, OpenSCAP is using a draft version of STIGs for Oracle Linux 9. -OpenScap can be configured to run at different time intervals. By default, OpenSCAP will run a remediation every 12 hours meaning any changes made to the system that bring it out of compliance will be reverted back to the STIG compliant state. This setting can be lowered or increased by modifying the `run_interval` setting found under [Administration](administration.md) --> Configuration --> stig +In order to apply STIGs on Security Onion we use a combination of our existing Saltstack configuration management and OpenSCAP. Currently, OpenSCAP is using a draft version of STIGs for Oracle Linux 9. + +OpenSCAP can be configured to run at different time intervals. By default, OpenSCAP will run a remediation every 12 hours meaning any changes made to the system that bring it out of compliance will be reverted back to the STIG compliant state. This setting can be lowered or increased by modifying the `run_interval` setting found under [Administration](administration.md) --> Configuration --> stig With the STIG feature enabled, you can find OpenSCAP reports under `/opt/so/log/stig`. Currently, the expected compliance score is 86%. diff --git a/docs/suricata.md b/docs/suricata.md index 3f0c28cb..9f558ad7 100644 --- a/docs/suricata.md +++ b/docs/suricata.md @@ -81,7 +81,7 @@ By default, Security Onion uses [Zeek](zeek.md) to record protocol metadata. If If you later find that some of that metadata is unnecessary, you can enable the SO_FILTERS ruleset to filter out unnecessary metadata. Navigate to [Administration](administration.md) --> Configuration --> SOC --> config --> server --> modules --> suricataengine --> rulesetSources and enable the SO_FILTERS ruleset. -To change your Grid's metadata engine from [Zeek](zeek.md) to Suricata, go to [Administration](administration.md) --> Configuration --> global --> mdengine and change the value from `Zeek` to `Suricata`: +To change your grid's metadata engine from [Zeek](zeek.md) to Suricata, go to [Administration](administration.md) --> Configuration --> global --> mdengine and change the value from `Zeek` to `Suricata`: ![Image](images/config-item-global.png) @@ -196,7 +196,9 @@ Values can be specified using the following formats (single or multi-line): Create custom variables by selecting an existing Address Group or Port Group and clicking "Duplicate". Enter a name using uppercase naming convention, then click "Create Setting". -**Note:** The new variable is not saved until you modify its value and click the green "Save Changes" checkmark. +**Note:** The new variable is not saved until you modify its value and click the green "Save Changes" checkmark. + +**Note:** Custom Port Group variables have some syntactical limitations, for example, not being able to use negation. ## Disabling @@ -206,4 +208,4 @@ If you need to disable Suricata, you can do so via [Administration](administrati !!! NOTE - For more information about Suricata, please see . \ No newline at end of file + For more information about Suricata, please see . diff --git a/docs/syslog.md b/docs/syslog.md index 9afb9e06..16950b8c 100644 --- a/docs/syslog.md +++ b/docs/syslog.md @@ -6,6 +6,6 @@ If your device does not have an existing [Elastic Agent](elastic-agent.md) integ ![Image](images/config-item-firewall.png) -Then choose the `syslog` option to allow the port through the firewall. If sending syslog to a sensor, please see the Examples in the [firewall](firewall.md) section. If you need to add custom parsing for those syslog logs, we recommend using [Elasticsearch](elasticsearch.md) ingest parsing. +Then choose the `syslog` option to allow the port through the firewall. If sending syslog to a sensor, please see the Examples in the [Firewall](firewall.md) section. If you need to add custom parsing for those syslog logs, we recommend using [Elasticsearch](elasticsearch.md) ingest parsing. -Also note that if you're monitoring network traffic with [Zeek](zeek.md), then by default it will detect any syslog in that network traffic and log it even if that syslog was not destined for that particular Security Onion node. \ No newline at end of file +Also note that if you're monitoring network traffic with [Zeek](zeek.md), then by default it will detect any syslog in that network traffic and log it even if that syslog was not destined for that particular Security Onion node. diff --git a/docs/telemetry.md b/docs/telemetry.md index dcbde9c4..7a806991 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -10,7 +10,7 @@ During setup, or during non-automated [soup](soup.md) invocations, the user must After installation, Grid administrators can enable or disable SOC Telemetry via the configuration interface. Search for `SOC Telemetry` in the Configuration screen. -After changing the `SOC Telemetry` configuration setting, the Grid must be resynchronized. This happens automatically once every 15 minutes, or manually if the Grid administrator clicks the Synchronize Grid option, at the top of the Configuration screen. Grid synchronization can take several minutes to complete. +After changing the `SOC Telemetry` configuration setting, the grid must be resynchronized. [Auto State Apply](salt.md#auto-state-apply) does this automatically within a few minutes. Grid synchronization can take several minutes to complete. Also, the browser will cache the previous configuration setting. Therefore, to ensure the browser is using the new setting value, make sure Grid synchronization completes, and then perform a hard browser refresh on the SOC UI. This can be performed via CTRL+SHIFT+R or CMD+SHIFT+R. @@ -45,7 +45,7 @@ Security Onion periodically checks for package updates to ensure the operating s Automatic package updates can be enabled or disabled via the configuration interface. Search for `patch` in the Configuration screen. -After changing this configuration setting, the Grid must be resynchronized. This happens automatically once every 15 minutes, or manually if the Grid administrator clicks the Synchronize Grid option, at the top of the Configuration screen. Grid synchronization can take several minutes to complete. +After changing this configuration setting, the grid must be resynchronized. [Auto State Apply](salt.md#auto-state-apply) does this automatically within a few minutes. Grid synchronization can take several minutes to complete. ### Included Data @@ -69,8 +69,8 @@ The telemetry data is sent using TLS encryption. ## Airgap -Grids installed within airgapped environments will automatically disable telemetry. In this scenario, the `SOC Telemetry` configuration setting will have no effect and the automatic package updates will be disabled. See the [airgap](airgap.md) section for more information about environments detached from the internet. +Grids installed within airgapped environments will automatically disable telemetry. In this scenario, the `SOC Telemetry` configuration setting will have no effect and the automatic package updates will be disabled. See the [Airgap](airgap.md) section for more information about environments detached from the internet. !!! NOTE - If a Grid is switched from airgap to non-airgap, and if the SOC Telemetry is not explicitly disabled in the Grid by an administrator, the SOC app running in the browser will send telemetry. \ No newline at end of file + If a grid is switched from airgap to non-airgap, and if the SOC Telemetry is not explicitly disabled in the grid by an administrator, the SOC app running in the browser will send telemetry. \ No newline at end of file diff --git a/docs/third-party-integrations.md b/docs/third-party-integrations.md index a8e0cd78..57d0d4e5 100644 --- a/docs/third-party-integrations.md +++ b/docs/third-party-integrations.md @@ -1,6 +1,6 @@ # Third Party Integrations -In addition to [Network](network-visibility.md) and [Host](host-visibility.md), you may want to pull in data from other third party systems. You can do that via Elastic integrations which support many of the most common products and services. You can read more about Elastic integrations at . +In addition to [network visibility](network-visibility.md) and [host visibility](host-visibility.md), you may want to pull in data from other third party systems. You can do that via Elastic integrations which support many of the most common products and services. You can read more about Elastic integrations at . !!! WARNING @@ -16,7 +16,7 @@ New integrations can be added to existing policies to provide increased visibili If an integration pulls the data, you should add it to the Fleet Server policy. Depending on complexity and log volume, it might make sense to stand up a Fleet Node and add your integrations to it. - If an integration receives data pushed to it (for example: receiving syslog), consider adding it to the Fleet Server policy. If that is not feasible, then you can add it to the Grid Nodes policy but make sure to set the firewall rules correctly so that you are not opening ports on all of your nodes. + If an integration receives data pushed to it (for example: receiving syslog), consider adding it to the Fleet Server policy. If that is not feasible, then you can add it to the grid Nodes policy but make sure to set the firewall rules correctly so that you are not opening ports on all of your nodes. To add an integration to an existing policy: @@ -51,7 +51,7 @@ To find integrations that have upgrades available: ## Managing Third Party Integration Index Templates -Index templates for third party integrations can be managed as described in the [Elasticsearch](elasticsearch.md) section, but first `managed_integrations` must be updated by navigating to [Administration](administration.md) --> Configuration --> Elasticsearch --> managed_integrations. +Index templates for third party integrations can be managed as described in the [Elasticsearch](elasticsearch.md) section, but first `managed_integrations` must be updated by navigating to [Administration](administration.md) --> Configuration --> manager --> managed_integrations. ## Supported Integrations @@ -61,4 +61,4 @@ The current release of Security Onion supports all standard Elastic integrations !!! NOTE - You can read more about Elastic integrations at . \ No newline at end of file + You can read more about Elastic integrations at . diff --git a/docs/tools.md b/docs/tools.md deleted file mode 100644 index 67728e5c..00000000 --- a/docs/tools.md +++ /dev/null @@ -1,20 +0,0 @@ -# Tools - -Security Onion would like to thank the following projects for their contribution to our community! - -(listed alphabetically) - -- [Attack Navigator](attack-navigator.md) -- [CyberChef](cyberchef.md) -- [Docker](docker.md) -- [ElastAlert](elastalert.md) -- [Elasticsearch](elasticsearch.md) -- [Elastic Agent](elastic-agent.md) -- [InfluxDB](influxdb.md) -- [Kibana](kibana.md) -- [Logstash](logstash.md) -- [Redis](redis.md) -- [Salt](salt.md) -- [Strelka](strelka.md) -- [Suricata](suricata.md) -- [Zeek](zeek.md) \ No newline at end of file diff --git a/docs/tricks-and-tips.md b/docs/tricks-and-tips.md index 1989ec82..0479f44b 100644 --- a/docs/tricks-and-tips.md +++ b/docs/tricks-and-tips.md @@ -1,18 +1,3 @@ -# Tricks and Tips +# Tricks and Tips Overview This section is a collection of miscellaneous tricks and tips for Security Onion. - -## Table of Contents - -- [Backup](backup.md) -- [Docker](docker.md) -- [Jupyter](jupyter.md) -- [New Disk](new-disk.md) -- [Network Installation](network-installation.md) -- [PCAPs](pcaps.md) -- [Performance](performance.md) -- [Removing a Node](removing-a-node.md) -- [Salt](salt.md) -- [Syslog Output](syslog-output.md) -- [Time Zones](time-zones.md) -- [Endgame](endgame.md) \ No newline at end of file diff --git a/docs/unifi.md b/docs/unifi.md index 04b576d4..ef0115a6 100644 --- a/docs/unifi.md +++ b/docs/unifi.md @@ -12,7 +12,7 @@ First, add the Elastic integrations for iptables and CEF. For more information about the CEF integration, see . -1. Go to [Elastic Fleet](elastic-fleet.md), click the `Agent policies` tab, and then click the desired policy (for example `so-Grid-nodes_general`). +1. Go to [Elastic Fleet](elastic-fleet.md), click the `Agent policies` tab, and then click the desired policy (for example `so-grid-nodes_general`). 2. Click the `Add integration` button. 3. Search for `iptables` and then click on the `iptables` integration. 4. The Elastic Integration page will show an overview of the iptables Integration. Review all information on the page and then click the `Add iptables` button. @@ -64,7 +64,7 @@ Finally, allow the traffic from the UniFi device through the Security Onion fire 3. On the left side, go to `firewall`, select `hostgroups`, and click the `customhostgroup0` group. On the right side, enter the IP address of the UniFi host and click the checkmark to save. 4. On the left side, go to `firewall`, select `portgroups`, select the `customportgroup0` group, and then click `udp`. On the right side, enter `9001` and `9003` and then click the checkmark to save. 5. On the left side, go to `firewall`, select `role`, and then select the node type that will receive the UniFi logs. Then drill into `chain` --> `INPUT` --> `hostgroups` --> `customhostgroup0` --> `portgroups`. On the right side, enter `customportgroup0` and click the checkmark to save. -6. If you would like to apply the rules immediately, click the `SYNCHRONIZE Grid` button under the `Options` menu at the top of the page. +6. If you would like to apply the rules immediately, click the `SYNCHRONIZE GRID` button under the `Options` menu at the top of the page. ## UniFi dashboards diff --git a/docs/updating.md b/docs/updating.md index ef73e640..25482490 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -1,8 +1,3 @@ -# Updating +# Updating Overview In this section, we'll cover keeping Security Onion up-to-date via [soup](soup.md) and list important [EOL](eol.md) dates for older versions of Security Onion. - -## Table of Contents - -- [soup](soup.md) -- [EOL](eol.md) \ No newline at end of file diff --git a/docs/use-cases.md b/docs/use-cases.md index 614420c2..0228898a 100644 --- a/docs/use-cases.md +++ b/docs/use-cases.md @@ -32,7 +32,7 @@ Suppose you have a small or medium network where you want some visibility for bo - Install the first Security Onion instance and choose the `ManagerSearch` option. - Deploy the [Elastic Agent](elastic-agent.md) to hosts. -- Install Security Onion on one or more additional machines and join them to the Grid as sensor nodes. They will analyze network traffic from your TAP or SPAN port. +- Install Security Onion on one or more additional machines and join them to the grid as sensor nodes. They will analyze network traffic from your TAP or SPAN port. You can read more about distributed deployments in the [Architecture](architecture.md) section. @@ -41,9 +41,9 @@ You can read more about distributed deployments in the [Architecture](architectu Suppose you have a medium or large network where you want some visibility for both network and hosts. A more scalable enterprise deployment would look like this: - Install the first Security Onion instance and choose the `Manager` option. -- Install Security Onion on one or more additional machines and join them to the Grid as search nodes. They will store logs and allow you to search them. -- Deploy the [Elastic Agent](elastic-agent.md) to hosts. They will collect logs and send them to the Grid. -- Install Security Onion on one or more additional machines and join them to the Grid as sensor nodes. They will analyze network traffic from your TAP or SPAN port. +- Install Security Onion on one or more additional machines and join them to the grid as search nodes. They will store logs and allow you to search them. +- Deploy the [Elastic Agent](elastic-agent.md) to hosts. They will collect logs and send them to the grid. +- Install Security Onion on one or more additional machines and join them to the grid as sensor nodes. They will analyze network traffic from your TAP or SPAN port. You can read more about distributed deployments in the [Architecture](architecture.md) section. @@ -52,11 +52,11 @@ You can read more about distributed deployments in the [Architecture](architectu Suppose you have a large network where you want maximum visibility for both network and hosts. A comprehensive distributed deployment would look like this: - Install the first Security Onion instance and choose the `Manager` option. -- Install Security Onion on one or more additional machines and join them to the Grid as search nodes. They will store logs and allow you to search them. -- Install Security Onion on a machine in your DMZ and join it to the Grid as a Fleet node. This node will manage your Elastic agents whether they are onsite or offsite. -- Deploy the [Elastic Agent](elastic-agent.md) to hosts. They will collect logs and send them to the Grid. -- Install Security Onion on one or more additional machines and join them to the Grid as sensor nodes. They will analyze network traffic from your TAP or SPAN port. -- Install Security Onion on one or more additional machines and join them to the Grid as receiver nodes. This provides load balancing and pipeline redundancy. -- Install Security Onion on one or more additional machines and join them to the Grid as [IDH](idh.md) nodes. They will provide honeypot and deception capabilities. +- Install Security Onion on one or more additional machines and join them to the grid as search nodes. They will store logs and allow you to search them. +- Install Security Onion on a machine in your DMZ and join it to the grid as a Fleet node. This node will manage your Elastic agents whether they are onsite or offsite. +- Deploy the [Elastic Agent](elastic-agent.md) to hosts. They will collect logs and send them to the grid. +- Install Security Onion on one or more additional machines and join them to the grid as sensor nodes. They will analyze network traffic from your TAP or SPAN port. +- Install Security Onion on one or more additional machines and join them to the grid as receiver nodes. This provides load balancing and pipeline redundancy. +- Install Security Onion on one or more additional machines and join them to the grid as [IDH](idh.md) nodes. They will provide honeypot and deception capabilities. You can read more about distributed deployments in the [Architecture](architecture.md) section. \ No newline at end of file diff --git a/docs/utilities.md b/docs/utilities.md index 736f410d..2b522d1d 100644 --- a/docs/utilities.md +++ b/docs/utilities.md @@ -1,16 +1,3 @@ -# Utilities +# Utilities Overview This section covers some of the utilities in Security Onion. - -## Table of Contents - -- [jq](jq.md) -- [so-allow](so-allow.md) -- [so-elastic-auth-password-reset](so-elastic-auth-password-reset.md) -- [so-elasticsearch-query](so-elasticsearch-query.md) -- [so-import-pcap](so-import-pcap.md) -- [so-import-evtx](so-import-evtx.md) -- [so-monitor-add](so-monitor-add.md) -- [so-status](so-status.md) -- [so-test](so-test.md) -- [so-user](so-user.md) \ No newline at end of file diff --git a/docs/virtualbox.md b/docs/virtualbox.md index 54fffe11..4154077a 100644 --- a/docs/virtualbox.md +++ b/docs/virtualbox.md @@ -5,7 +5,7 @@ In this section, we'll cover installing Security Onion on VirtualBox. You can do ## Creating VM - Launch VirtualBox and click the `New` button. -- Provide a name for the virtual machine (`Security Onion 2.4` for example) and then select the ISO image. It should automatically set type to `Linux` and version to `Oracle Linux 9.x`. Click the checkbox for `Skip Unattended Installation` and then click the `Next` button. +- Provide a name for the virtual machine (`Security Onion` for example) and then select the ISO image. It should automatically set type to `Linux` and version to `Oracle Linux 9.x`. Click the checkbox for `Skip Unattended Installation` and then click the `Next` button. - Specify RAM and Processors as needed per the [Hardware](hardware.md) section and then click the `Next` button. - Specify virtual hard disk size as needed per the [Hardware](hardware.md) section and then click the `Next` button. - Confirm options and then click the `Finish` button. diff --git a/docs/yara.md b/docs/yara.md index 6306cae2..6ca8f6fe 100644 --- a/docs/yara.md +++ b/docs/yara.md @@ -24,7 +24,7 @@ To add a new YARA rule, go to the main [Detections](detections.md) page and clic 1. Click the Language drop-down and select `YARA`. 2. Optionally specify a license. 3. Add the signature. -4. Click the `CREATE` button and the detection should deploy to your Grid at the next 15-minute cycle. +4. Click the `CREATE` button and [Auto State Apply](salt.md#auto-state-apply) should deploy the detection to your grid within a few minutes. ![Image](images/59_detection_create.png) diff --git a/docs/zeek-fields.md b/docs/zeek-fields.md index c3d785c3..f7049492 100644 --- a/docs/zeek-fields.md +++ b/docs/zeek-fields.md @@ -13,16 +13,16 @@ The remaining fields in each log are specific to the log type. To see how the fi You can find ingest parsers in your local filesystem at `/opt/so/conf/elasticsearch/ingest/` or you can find them online at: - + For example, suppose you want to know how the Zeek conn.log is parsed. You could take a look at `/opt/so/conf/elasticsearch/ingest/zeek.conn` or view it online at: - + You'll see that `Zeek.conn` then calls the `Zeek.common` pipeline (`/opt/so/conf/elasticsearch/ingest/zeek.common`): - + which in turn calls the `common` pipeline (`/opt/so/conf/elasticsearch/ingest-dynamic/common`): - \ No newline at end of file + diff --git a/docs/zeek.md b/docs/zeek.md index a20db4ee..caef1b23 100644 --- a/docs/zeek.md +++ b/docs/zeek.md @@ -117,23 +117,45 @@ Please note that Zeek is very strict about the format of `intel.dat`. When editi The default `intel.dat` file follows these guidelines so you can reference it as an example of the proper format. -When finished editing `intel.dat`, run `sudo salt $SENSORNAME_$ROLE state.highstate` to sync `/opt/so/saltstack/local/salt/zeek/policy/intel/` to `/opt/so/conf/zeek/policy/intel/`. If you have a distributed deployment with separate sensor nodes, it may take up to 15 minutes for intel to sync to the sensor nodes. +[Auto State Apply](salt.md#auto-state-apply) picks up changes to `intel.dat` within a few minutes. If you don't want to wait, run `sudo salt -C 'I@zeek:enabled:true' state.apply zeek` to sync `/opt/so/saltstack/local/salt/zeek/policy/intel/` to `/opt/so/conf/zeek/policy/intel/`. Either way, intel syncs to every node running Zeek, including separate sensor nodes in a distributed deployment. -If you experience an error, or do not notice `/nsm/zeek/logs/current/intel.log` being generated, try having a look in `/nsm/zeek/logs/current/reporter.log` for clues. You may also want to restart Zeek after making changes by running `sudo so-Zeek-restart`. +If you experience an error, or do not notice `/nsm/zeek/logs/current/intel.log` being generated, try having a look in `/nsm/zeek/logs/current/reporter.log` for clues. You may also want to restart Zeek after making changes by running `sudo so-zeek-restart`. For more information, please see . +## Custom Packages + +You can install custom Zeek packages using `zkg`. Place each package as a subdirectory in `/opt/so/saltstack/local/salt/zeek/zkg/` on the manager. For example, if you have a custom package called `my-package`, the directory structure would be: + +``` +/opt/so/saltstack/local/salt/zeek/zkg/my-package/ +``` + +The package directory should contain a valid Zeek package structure (including a `zkg.meta` file). + +!!! NOTE + + zkg requires packages from git repos to be a clone of the repo. The working tree must be clean otherwise it will not install the package. `git status` will tell you if it is clean or not. + +[Auto State Apply](salt.md#auto-state-apply) picks up new packages within a few minutes. If you don't want to wait, run `sudo salt -C 'I@zeek:enabled:true' state.apply zeek` to sync the packages to the sensor nodes. The packages will be automatically installed with `zkg` each time the Zeek container starts. + +You can verify that a custom package was installed by checking the Zeek container logs: + +``` +sudo docker logs so-zeek +``` + ## Diagnostic Logging Zeek diagnostic logs can be found in `/nsm/zeek/logs/`. Look for files like `reporter.log`, `stats.log`, `stderr.log`, and `stdout.log`. Depending on what you're looking for, you may also need to look at the [Docker](docker.md) logs for the container: ``` -sudo docker logs so-Zeek +sudo docker logs so-zeek ``` ## More Information !!! NOTE - For more information about Zeek, please see . \ No newline at end of file + For more information about Zeek, please see . diff --git a/mkdocs.yml b/mkdocs.yml index e01353b6..03a9cf85 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,16 +1,15 @@ site_name: Security Onion Documentation -site_url: https://security-onion-solutions.github.io/securityonion-3-docs/ -site_author: Security Onion Solutions, LLC +site_url: https://security-onion-solutions.github.io/docs/ +site_author: "" copyright: Security Onion Solutions, LLC theme: name: material logo: images/logo/so-logo.png - favicon: images/logo/so-logo.png + favicon: images/logo/favicon.ico features: - navigation.instant - navigation.tracking - - navigation.expand - navigation.path - navigation.top - navigation.footer @@ -39,7 +38,7 @@ extra: - icon: fontawesome/brands/twitter link: https://twitter.com/securityonion - icon: fontawesome/brands/youtube - link: https://www.youtube.com/c/SecurityOnion + link: https://securityonion.com/youtube extra_css: - _static/theme_overrides.css @@ -50,6 +49,11 @@ extra_javascript: plugins: - search - glightbox + - to-pdf: + cover_subtitle: !ENV [VERSION, ''] + cover_logo: "https://securityonionsolutions.com/logo/logo-so-onion-light.svg" + output_path: securityonion-docs.pdf + download_link: header markdown_extensions: - admonition @@ -61,21 +65,17 @@ markdown_extensions: anchor_linenums: true - pymdownx.inlinehilite - pymdownx.snippets - - pymdownx.emoji: - emoji_index: !!python/name:material.extensions.emoji.twemoji - emoji_generator: !!python/name:material.extensions.emoji.to_svg - toc: permalink: true toc_depth: 3 nav: - - Home: index.md - - About: about.md + - About: index.md - Introduction: introduction.md - License: license.md - First Time Users: first-time-users.md - Getting Started: - - Overview: getting-started.md + - Getting Started Overview: getting-started.md - Best Practices: best-practices.md - Use Cases: use-cases.md - Architecture: architecture.md @@ -93,7 +93,7 @@ nav: - Configuration: configuration.md - Post Installation: post-installation.md - Security Onion Console: - - Overview: security-onion-console.md + - Security Onion Console Overview: security-onion-console.md - Alerts: alerts.md - Dashboards: dashboards.md - Hunt: hunt.md @@ -107,15 +107,16 @@ nav: - Elastic Fleet: elastic-fleet.md - Osquery Manager: osquery-manager.md - InfluxDB: influxdb.md + - PostgreSQL: postgresql.md - CyberChef: cyberchef.md - Attack Navigator: attack-navigator.md - Security Onion Desktop: - - Overview: security-onion-desktop.md + - Security Onion Desktop Overview: security-onion-desktop.md - Chromium: chromium.md - NetworkMiner: networkminer.md - Wireshark: wireshark.md - Network Visibility: - - Overview: network-visibility.md + - Network Visibility Overview: network-visibility.md - AF-PACKET: af-packet.md - BPF: bpf.md - Full Packet Capture: full-packet-capture.md @@ -124,7 +125,7 @@ nav: - Strelka: strelka.md - IDH: idh.md - Additional Network Visibility: - - Overview: additional-network-visibility.md + - Additional Network Visibility Overview: additional-network-visibility.md - NetFlow: netflow.md - CEF: cef.md - iptables: iptables.md @@ -132,18 +133,19 @@ nav: - pfSense: pfsense.md - OPNsense: opnsense.md - Host Visibility: - - Overview: host-visibility.md + - Host Visibility Overview: host-visibility.md - Elastic Agent: elastic-agent.md - Syslog: syslog.md - Sysmon: sysmon.md - Third Party Integrations: third-party-integrations.md - Rules: - - Overview: rules.md + - Rules Overview: rules.md - NIDS: nids.md - Sigma: sigma.md + - Sigma Logsource Coverage: sigma-logsource-coverage.md - YARA: yara.md - Logs: - - Overview: logs.md + - Logs Overview: logs.md - Ingest: ingest.md - Logstash: logstash.md - Redis: redis.md @@ -156,11 +158,11 @@ nav: - Community ID: community-id.md - Security Onion Console Logs: security-onion-console-logs.md - Updating: - - Overview: updating.md + - Updating Overview: updating.md - soup: soup.md - EOL: eol.md - Accounts: - - Overview: accounts.md + - Accounts Overview: accounts.md - Adding Accounts: adding-accounts.md - Disabling Accounts: disabling-accounts.md - Listing Accounts: listing-accounts.md @@ -170,24 +172,24 @@ nav: - Kratos: kratos.md - Services: services.md - Customizing: - - Overview: customizing.md + - Customizing Overview: customizing.md - Security Onion Console Customization: security-onion-console-customization.md - nginx: nginx.md - - proxy: proxy.md - - firewall: firewall.md - - email: email.md - - ntp: ntp.md - - console: console.md - - ssh: ssh.md - - hostname: hostname.md - - ip: ip.md - - dns: dns.md + - Proxy: proxy.md + - Firewall: firewall.md + - Email: email.md + - NTP: ntp.md + - SSH: ssh.md + - Hostname: hostname.md + - IP Address: ip-address.md + - DNS: dns.md - URL Base: url-base.md - Tricks and Tips: - - Overview: tricks-and-tips.md + - Tricks and Tips Overview: tricks-and-tips.md - Backup: backup.md - Docker: docker.md - Jupyter: jupyter.md + - Kernels: kernels.md - New Disk: new-disk.md - Network Installation: network-installation.md - PCAPs: pcaps.md @@ -196,11 +198,11 @@ nav: - Salt: salt.md - Syslog Output: syslog-output.md - Time Zones: time-zones.md - - Endgame: endgame.md - Utilities: - - Overview: utilities.md + - Utilities Overview: utilities.md - jq: jq.md - so-allow: so-allow.md + - so-detections-overrides-import: so-detections-overrides-import.md - so-elastic-auth-password-reset: so-elastic-auth-password-reset.md - so-elasticsearch-query: so-elasticsearch-query.md - so-import-pcap: so-import-pcap.md @@ -210,22 +212,21 @@ nav: - so-test: so-test.md - so-user: so-user.md - Help: - - Overview: help.md + - Help Overview: help.md - FAQ: faq.md - Directory: directory.md - - Tools: tools.md - Community Support: community-support.md - Support: support.md - Help Wanted: help-wanted.md - Security Onion Pro: - - Overview: security-onion-pro.md + - Security Onion Pro Overview: security-onion-pro.md - OIDC: oidc.md - LUKS: luks.md - FIPS: fips.md - STIG: stig.md - Notifications: notifications.md - Kafka: kafka.md - - Connect API: connect-api.md + - Security Onion API: connect-api.md - Active Query Management: active-query-management.md - Manager of Managers: manager-of-managers.md - MCP Server: mcp-server.md @@ -233,8 +234,10 @@ nav: - Hypervisor: hypervisor.md - Reports: reports.md - Onion AI: onion-ai.md - - Security: security.md + - Local LLM Hosting: local-llm.md - Telemetry: telemetry.md + - Security: security.md + - Software Bill of Materials: software-bill-of-materials.md - Release Notes: release-notes.md - Appendix: appendix.md - Cheat Sheet: cheat-sheet.md diff --git a/specs/openapi.yaml b/specs/openapi.yaml index 0ccebfb7..be43a7a9 100644 --- a/specs/openapi.yaml +++ b/specs/openapi.yaml @@ -61,6 +61,54 @@ components: example: 0 type: integer type: object + model.AdapterParameters: + properties: + name: + type: string + protocol: + type: string + supportsEmbeddings: + type: boolean + type: object + model.Agent: + properties: + agentDescription: + example: Analyzes suspicious binaries and scripts + type: string + allowedSkills: + example: + - Hunt + - Respond + items: + type: string + type: array + uniqueItems: false + canDelegateTo: + example: + - Log Analyst + items: + type: string + type: array + uniqueItems: false + enabled: + description: A disabled agent is still published so the Agent Studio can + re-enable it. + example: true + type: boolean + isOrchestrator: + example: false + type: boolean + isSystem: + example: true + type: boolean + name: + example: Malware Analyst + type: string + personaAddendum: + description: Admin-authored persona; exposed because an admin wrote it. + example: Prefer static analysis before detonating a sample. + type: string + type: object model.AlertingParameters: properties: ackEnabled: @@ -134,6 +182,27 @@ components: example: 132 type: integer type: object + model.AllocationDecider: + properties: + explanation: + description: The datastore's explanation of why the rule rejected or throttled + the shard. + example: the node is above the low watermark cluster setting + type: string + name: + description: The name of the allocation rule. + example: disk_threshold + type: string + nodes: + description: The names of the nodes the rule rejected or throttled the shard + on. + example: + - so-node-01 + items: + type: string + type: array + uniqueItems: false + type: object model.Artifact: properties: artifactType: @@ -257,36 +326,102 @@ components: type: object model.AssistantParameters: properties: + agentMapping: + additionalProperties: + type: string + example: + Malware Analyst: claude-sonnet-4.5@SOAI + type: object + agentic: + type: boolean + availableAdapters: + items: + $ref: '#/components/schemas/model.AdapterParameters' + type: array + uniqueItems: false + availableAgents: + items: + $ref: '#/components/schemas/model.Agent' + type: array + uniqueItems: false availableModels: items: $ref: '#/components/schemas/model.ModelParameters' type: array uniqueItems: false + availableSkills: + items: + $ref: '#/components/schemas/model.Skill' + type: array + uniqueItems: false + availableTools: + description: Tool names an admin-created skill may grant; delegate tools + excluded. + example: + - query_events + - query_cases + items: + type: string + type: array + uniqueItems: false compressContextPrompt: type: string enabled: type: boolean investigationPrompt: type: string + maxDelegationDepth: + description: |- + Delegation guardrails, surfaced so the Agent Studio can show and edit them + without fetching every setting. 0 disables the limit. + example: 3 + type: integer + maxSubSessionTokens: + example: 100000 + type: integer + memoryEnabled: + type: boolean + memoryParams: + $ref: '#/components/schemas/model.MemoryParameters' thresholdColorRatioLow: type: number thresholdColorRatioMax: type: number thresholdColorRatioMed: type: number + toolBusyMaxRetries: + type: integer + toolBusyRetryDelayMs: + type: integer type: object model.AssistantSession: - description: A session for a user chatting with the Assistant. + description: Meta information about the session. properties: createTime: description: The date and time that this object was created. This is a read-only field. example: "2024-11-14T15:03:22Z" type: string + delegateAgent: + description: For delegated sub-agent sessions, the display name of the delegated + agent. + example: Hunter + type: string deleteTime: description: The time the session was deleted. example: "2025-09-05T15:33:00.000Z" type: string + depth: + description: |- + The delegation nesting depth of this session: 0 for a top-level conversation, + parent depth + 1 for a delegated sub-agent. Used to enforce a delegation depth + limit. Absent (0) for legacy sessions created before this field existed. + example: 1 + type: integer + entityId: + description: The entity ID associated with this session (e.g., alert soc_id). + example: WKhCuTw4GPvrQA-9ksmn + type: string id: description: The ID assigned to this object by the server. This is a read-only field. @@ -296,11 +431,52 @@ components: description: The kind of object. This is a read-only field. example: case type: string + lastMemoryScannedIndex: + description: |- + When Memory is enabled, the scanner will use this field to denote how many + messages in the session have been scanned for memory extraction. Extraction + always begins with the oldest messages in a session. + example: 4 + type: integer + messageCount: + description: |- + A denormalized count of the messages saved to this session, incremented on + each chat save so the memory scanner can find sessions with unscanned + messages in a single query. Not omitempty so new sessions index an explicit + 0; sessions created before this field existed have no value, which the + scanner treats as pending until its index update heals it. + example: 7 + type: integer + model: + description: |- + The model/adapter this session itself runs on (the agent id for delegated + sessions). Used to resume the session server-side without trusting the + client-supplied model. Empty for legacy sessions created before this field + existed, in which case the caller falls back to the request/parent model. + example: AgentClaude@SOAI + type: string operation: description: The operation that was applied to the object. This is a read-only field. example: create type: string + parentModel: + description: |- + For delegated sub-agent sessions, the model the parent session uses, so the + parent can be resumed once the sub-agent's result is ready. + example: AgentClaude@SOAI + type: string + parentSessionId: + description: For delegated sub-agent sessions, the session that delegated + to this one. + example: chat_1757086398900_ykhmndscn + type: string + parentToolUseId: + description: |- + For delegated sub-agent sessions, the tool_use id in the parent session that + this delegation resolves when the sub-agent finishes. + example: tooluse_mT45or7ISwSEUivo63nqow + type: string sessionId: description: The session identifier. example: chat_1757086398900_ykhmndscn @@ -318,6 +494,10 @@ components: the user. example: Can you write a suricata rule for me? type: string + type: + description: The type of session (e.g., "alert_investigation"). + example: alert_investigation + type: string updateTime: description: The date and time that this object was last modified. This is a read-only field. @@ -331,6 +511,29 @@ components: example: socl_my_new_client type: string type: object + model.AssistantSessionDetails: + description: Detailed information about an Assistant session, including its + messages. + properties: + history: + description: The messages in the session. + items: + $ref: '#/components/schemas/model.StoredMessage' + type: array + uniqueItems: false + pendingApproval: + $ref: '#/components/schemas/model.PendingToolApproval' + session: + $ref: '#/components/schemas/model.AssistantSession' + subSessions: + description: |- + Delegated sub-sessions descending from this session (any depth), each with + their own messages. Used to reconstruct nested sub-agent activity on reload. + items: + $ref: '#/components/schemas/model.AssistantSessionDetails' + type: array + uniqueItems: false + type: object model.AttachEventCriteria: properties: acknowledged: @@ -372,6 +575,42 @@ components: - caseId - fields type: object + model.Attachment: + description: Attachment represents a file or graph payload to be sent with a + notification. + properties: + contentType: + description: The MIME content type of the attachment. + example: application/pdf + type: string + filename: + description: The filename of the attachment. + example: alert-report.pdf + type: string + url: + description: Direct download URL if the attachment is hosted by SOC. + example: https://soc.example.com/api/reports/123/download + type: string + type: object + model.AuditHistory: + properties: + duplicatedFromId: + type: string + newValue: + type: string + nodeId: + type: string + note: + type: string + oldValue: + type: string + settingId: + type: string + timestamp: + type: string + userId: + type: string + type: object model.Auditable: description: The Auditable fields are read-only. They are generated by the server. properties: @@ -963,6 +1202,11 @@ components: model.EngineState: description: The state of the Strelka detection engine properties: + blocked: + description: True if sync operations are currently blocked (e.g., during + migrations). + example: false + type: boolean importing: description: True if new detections are currently being imported into the engine. @@ -1150,6 +1394,10 @@ components: type: object description: The collection of aggregated metrics associated with this search type: object + timedOut: + description: Did the query that produced these results time out while collecting + them + type: boolean totalEvents: description: The total number of matching events (not necessarily the total number returned) @@ -1187,6 +1435,12 @@ components: description: The maximum number of metrics to limit in aggregate groups example: 10 type: integer + params: + additionalProperties: {} + description: Any parameters used in the event update scripts + example: + '{"userId"': ' "admin"}' + type: object query: description: The base query used to conduct the event search example: (*) AND tags:alert AND NOT event.acknowledged:true AND NOT event.escalated:true @@ -1226,6 +1480,13 @@ components: type: string type: array uniqueItems: false + taskIds: + description: The Elasticsearch task ids when the update runs asynchronously + (one per host); empty for synchronous updates + items: + type: string + type: array + uniqueItems: false unchangedCount: description: The number of events the were left unmodified example: 0 @@ -1235,6 +1496,102 @@ components: example: 1 type: integer type: object + model.EventsHealth: + properties: + errors: + additionalProperties: + type: string + description: The sections that could not be collected, keyed by section + name, with the reason for each failure. + example: + nodes: unavailable + type: object + indicators: + description: The per-subsystem health checks reported by the datastore, + most severe first. + items: + $ref: '#/components/schemas/model.HealthIndicator' + type: array + uniqueItems: false + nodes: + description: The nodes making up the datastore cluster. + items: + $ref: '#/components/schemas/model.EventsHealthNode' + type: array + uniqueItems: false + settings: + $ref: '#/components/schemas/model.EventsHealthSettings' + status: + description: 'The overall datastore health: green, yellow, red, or unknown.' + example: yellow + type: string + unassignedShards: + $ref: '#/components/schemas/model.UnassignedShards' + type: object + model.EventsHealthNode: + properties: + cpu: + description: The percentage of the node's CPU in use. + example: "12" + type: string + diskTotal: + description: The total size of the node's data storage. + example: 1.5tb + type: string + diskUsedPercent: + description: The percentage of the node's data storage in use. + example: "85.5" + type: string + heapPercent: + description: The percentage of the node's heap in use. + example: "62" + type: string + ip: + description: The IP address the node is bound to. + example: 10.0.0.5 + type: string + load1m: + description: The node's one minute load average. + example: "1.42" + type: string + master: + description: An asterisk if the node is the elected master, otherwise a + dash. + example: '*' + type: string + name: + description: The node name. + example: so-node-01 + type: string + ramPercent: + description: The percentage of the node's memory in use. + example: "88" + type: string + roles: + description: The abbreviated roles the node performs in the cluster. + example: dhimrst + type: string + uptime: + description: How long the node has been running. + example: 42d + type: string + version: + description: The datastore version the node is running. + example: 8.14.3 + type: string + type: object + model.EventsHealthSettings: + description: The cluster settings that have been overridden from their defaults. + properties: + persistent: + additionalProperties: {} + description: The settings that survive a cluster restart. + type: object + transient: + additionalProperties: {} + description: The settings that are discarded on a cluster restart. + type: object + type: object model.Filter: description: Optional filter for the job, typically used for packet filtering properties: @@ -1287,6 +1644,23 @@ components: example: 55312 type: integer type: object + model.FindingScope: + description: The group of shards this finding was diagnosed from, when it came + from the shard inventory. + properties: + count: + description: The number of shards in the group. + example: 2 + type: integer + primary: + description: Whether the group contains primary shards rather than replicas. + example: true + type: boolean + reason: + description: The datastore's reason code for why the shards became unassigned. + example: NODE_LEFT + type: string + type: object model.GridMember: properties: fingerprint: @@ -1316,6 +1690,8 @@ components: properties: maxUploadSize: type: integer + metricsDashboard: + $ref: '#/components/schemas/model.MetricsDashboard' staleMetricsMs: type: integer type: object @@ -1340,6 +1716,87 @@ components: example: 0 type: integer type: object + model.HealthFinding: + properties: + condition: + description: The identifier for the problem that was diagnosed. + example: disk_threshold + type: string + count: + description: The number of affected resources, when the condition is counted + rather than scoped. + example: 3 + type: integer + detail: + description: The datastore's own explanation of the condition. + example: the node is above the low watermark cluster setting + type: string + nodes: + description: The names of the nodes affected by this condition. + example: + - so-node-01 + items: + type: string + type: array + uniqueItems: false + scope: + $ref: '#/components/schemas/model.FindingScope' + severity: + description: 'The importance of this finding: critical, warning, or info.' + example: critical + type: string + type: object + model.HealthIndicator: + properties: + causes: + description: The datastore's diagnoses of the symptom, along with the resources + each affects. + items: + $ref: '#/components/schemas/model.HealthIndicatorCause' + type: array + uniqueItems: false + findings: + description: Diagnosed problems for this indicator, most severe first + items: + $ref: '#/components/schemas/model.HealthFinding' + type: array + uniqueItems: false + id: + description: The datastore's identifier for the subsystem being checked. + example: shards_availability + type: string + status: + description: 'The health of this subsystem: green, yellow, red, or unknown.' + example: red + type: string + symptom: + description: The datastore's summary of what is wrong with this subsystem. + example: This cluster has 1 unavailable primary shard. + type: string + type: object + model.HealthIndicatorCause: + properties: + cause: + description: The datastore's description of the cause. + example: A node has recently left the cluster. + type: string + indices: + description: The names of the indices affected by this cause. + example: + - so-logs-2026.07.21 + items: + type: string + type: array + uniqueItems: false + nodes: + description: The names of the nodes affected by this cause. + example: + - so-node-01 + items: + type: string + type: array + uniqueItems: false + type: object model.HuntingAction: properties: background: @@ -1473,6 +1930,11 @@ components: forceUserOtp: description: OTP indicator; unavailable to API clients type: boolean + lastUnreadNotificationTime: + description: The timestamp of the most recent unread notification for the + user. + example: "2026-08-20T17:00:00Z" + type: string license: description: The copyright license applicable to the Security Onion software example: Elastic License 2.0 (ELv2) @@ -1488,6 +1950,10 @@ components: description: The MAC address assigned to the management interface. example: 11:22:33:AA:BB:CC type: string + notificationsStarted: + description: Indicates whether the notification subsystem module is started + and available. + type: boolean parameters: $ref: '#/components/schemas/model.ClientParameters' srvToken: @@ -1600,44 +2066,257 @@ components: example: no result type: string type: object - model.Message: - description: The message content. + model.MemoryParameters: properties: - id: - description: The unique identifier for the message. Not required. - example: c3d44fb8-3bc2-46e2-a7d2-8a8983556d1a - type: string - role: - description: Indicates who authored the message. Either `user` or `assistant`. - example: user - type: string - stop_reason: - description: The reason the message was stopped. - example: user_request + dontScanBefore: type: string - stop_sequence: - description: The sequence in which the message was stopped. - example: end_turn + embedModel: + example: amazon.titan-embed-text-v2@SOAI type: string - usage: - $ref: '#/components/schemas/model.Usage' - type: object - model.ModelParameters: - properties: - contextLimitLarge: + maxGlobalMemoriesToInclude: + example: 5 type: integer - contextLimitSmall: + maxGlobalMemoriesToReconcile: + example: 20 type: integer - displayName: + maxUserMemoriesToInclude: + example: 5 + type: integer + maxUserMemoriesToReconcile: + example: 20 + type: integer + memoryModel: + example: sonnet@SOAI type: string - enabled: - type: boolean - id: + memoryPersona: type: string - lowBalanceColorAlert: + memoryProximityThreshold: + example: 0.8 + type: number + messageProximityThreshold: + example: 0.5 + type: number + reconcileModel: + example: sonnet@SOAI + type: string + reconcilePersona: + type: string + scanIntervalSeconds: + example: 300 type: integer + staleMemoryCount: + type: integer + useMemory: + example: true + type: boolean + useMemoryScanner: + example: true + type: boolean type: object - model.Node: + model.MemoryRecord: + properties: + createTime: + type: string + id: + example: c3d44fb8-3bc2-46e2-a7d2-8a8983556d1a + type: string + lastUsedAt: + type: string + memoryText: + example: The user prefers timestamps in UTC. + type: string + modelId: + example: claude-sonnet-4.5 + type: string + scope: + example: user + type: string + sessionId: + example: chat_1757086398900_ykhmndscn + type: string + similarity: + example: 0.82 + type: number + targetUserId: + example: 8beae4b5-275b-4669-b678-8cff894911b5 + type: string + updateTime: + type: string + usageCount: + example: 4 + type: integer + userDefined: + example: true + type: boolean + type: object + model.MemoryRequest: + properties: + memoryText: + example: The user prefers timestamps in UTC. + type: string + scope: + example: user + type: string + targetUserId: + description: |- + TargetUserId assigns a user-scoped memory to someone else; requires + memory/write_all. Empty means the requestor. + example: 8beae4b5-275b-4669-b678-8cff894911b5 + type: string + type: object + model.MemoryResults: + properties: + limit: + example: 25 + type: integer + memories: + items: + $ref: '#/components/schemas/model.MemoryRecord' + type: array + uniqueItems: false + offset: + example: 0 + type: integer + total: + example: 42 + type: integer + type: object + model.Message: + description: The message content. + properties: + id: + description: The unique identifier for the message. Not required. + example: c3d44fb8-3bc2-46e2-a7d2-8a8983556d1a + type: string + role: + description: Indicates who authored the message. Either `user` or `assistant`. + example: user + type: string + stop_reason: + description: The reason the message was stopped. + example: user_request + type: string + stop_sequence: + description: The sequence in which the message was stopped. + example: end_turn + type: string + thoughts: + description: The plain text thoughts from the model's response + type: string + usage: + $ref: '#/components/schemas/model.Usage' + type: object + model.MetricPanel: + properties: + colors: + example: + - '["#4dc9f6"]' + items: + type: string + type: array + uniqueItems: false + height: + example: 250 + type: integer + id: + example: cpu + type: string + keys: + example: + - '["cpu_used"]' + items: + type: string + type: array + uniqueItems: false + labelKeys: + example: + - '["cpu"]' + items: + type: string + type: array + uniqueItems: false + labels: + example: + - '["CPU"]' + items: + type: string + type: array + uniqueItems: false + metric: + example: cpu + type: string + title: + example: CPU Usage + type: string + titleKey: + example: metricsCpuUsage + type: string + type: + example: Xy + type: string + units: + example: '%' + type: string + width: + example: 6 + type: integer + type: object + model.MetricSample: + properties: + timestamp: + type: string + value: + type: number + type: object + model.MetricsDashboard: + properties: + panels: + items: + $ref: '#/components/schemas/model.MetricPanel' + type: array + uniqueItems: false + type: object + model.ModelParameters: + properties: + adapter: + type: string + charsPerTokenEstimate: + type: number + contextLimitLarge: + type: integer + contextLimitSmall: + type: integer + displayName: + type: string + enabled: + type: boolean + id: + type: string + lowBalanceColorAlert: + type: integer + origin: + type: string + type: object + model.ModelUsageStats: + properties: + modelCredits: + description: The total credits used for this model. + example: 2 + type: integer + modelInputTokens: + description: The total input tokens used for this model. + example: 500 + type: integer + modelMessages: + description: The total messages sent using this model. + example: 10 + type: integer + modelOutputTokens: + description: The total output tokens used for this model. + example: 1000 + type: integer + type: object + model.Node: properties: address: description: The IP address of this node @@ -1724,6 +2403,10 @@ components: node example: 220 type: integer + historicalMetricsEnabled: + description: Indicates whether historical time-series metrics are supported/enabled + example: true + type: boolean id: description: The node ID example: sensor-001 @@ -1744,6 +2427,10 @@ components: description: Indicates whether disk encryption is enabled on this node example: 1 type: integer + load5m: + description: The node's 5-minute load metric + example: 0.96 + type: number load15m: description: The node's 15-minute load metric example: 1.09 @@ -1752,10 +2439,6 @@ components: description: The node's 1-minute load metric example: 0.43 type: number - load5m: - description: The node's 5-minute load metric - example: 0.96 - type: number memoryTotalGB: description: Total size, in gigabytes, of the system memory, or RAM example: 33.176731648 @@ -1834,16 +2517,30 @@ components: restart' example: ok type: string - stenoLossPct: - description: The current percentage packet loss experienced by Stenographer, - if applicable to this node - example: 0 - type: number suriLossPct: description: The current percentage packet loss experienced by Suricata, if applicable to this node example: 1.1345 type: number + suriRulesFailed: + description: The number of Suricata rules that failed to load, if applicable + to this node + example: 0 + type: integer + suriRulesLoaded: + description: The number of Suricata rules currently loaded, if applicable + to this node + example: 45879 + type: integer + suriRulesReloadTime: + description: The timestamp of the last Suricata rule reload, if applicable + to this node + example: 2025-12-04T01:04:07.994734+0000 + type: string + suriRulesStatus: + description: 'The status of Suricata rule loading: ok, unknown' + example: ok + type: string swapTotalGB: description: Total size, in gigabytes, of the system swap memory example: 8.589930496000001 @@ -1891,6 +2588,115 @@ components: example: 0 type: number type: object + model.NotificationAuditEntry: + description: NotificationAuditEntry represents a user view or dismissal audit + state record. + properties: + dismissedAt: + description: The timestamp when this user dismissed the notification. + example: "2026-08-17T12:10:00Z" + type: string + isDismissed: + description: Indicates whether this user has dismissed the notification. + example: false + type: boolean + isRead: + description: Indicates whether this user has read the notification. + example: true + type: boolean + readAt: + description: The timestamp when this user read the notification. + example: "2026-08-17T12:05:00Z" + type: string + userId: + description: The user identifier who read or dismissed the notification. + example: admin@soc.local + type: string + type: object + model.NotificationRecord: + description: NotificationRecord represents a persisted notification with user-specific + state. + properties: + attachments: + description: Optional binary or URL attachments associated with this notification. + items: + $ref: '#/components/schemas/model.Attachment' + type: array + uniqueItems: false + createdAt: + description: The timestamp when this notification was created. + example: "2026-08-17T12:00:00Z" + type: string + dismissedAt: + description: The timestamp when the notification was dismissed. + example: "2026-08-17T12:10:00Z" + type: string + fields: + additionalProperties: + type: string + description: Structured key-value fields providing context (e.g. host, IP, + metric name). + example: + dst_ip: 10.0.0.1 + src_ip: 192.168.1.100 + type: object + id: + description: The unique identifier for this notification. + example: a1b2c3d4-e5f6-7890-abcd-ef1234567890 + type: string + isDismissed: + description: Indicates whether this notification has been dismissed by the + user. + example: false + type: boolean + isRead: + description: Indicates whether this notification has been marked as read + by the user. + example: false + type: boolean + links: + additionalProperties: + type: string + description: Deep-links back to relevant views or dashboards in SOC. + example: + View in SOC: https://soc.example.com/#/alerts/123 + type: object + readAt: + description: The timestamp when the notification was marked as read. + example: "2026-08-17T12:05:00Z" + type: string + severity: + description: The severity classification of this notification. + enum: + - info + - low + - medium + - high + - critical + example: high + type: string + silenceKey: + description: Unique key used for duration silencing and debouncing. + example: detection:sigma:12345:192.168.1.100 + type: string + source: + description: The source subsystem that originated this notification. + enum: + - detection + - metric + - agent_ai + - report + example: detection + type: string + summary: + description: A human-readable summary describing the notification details. + example: Inbound SSH scan detected from 192.168.1.100. + type: string + title: + description: The brief title or headline of the notification. + example: ET SCAN Potential SSH Scan + type: string + type: object model.Override: properties: count: @@ -2003,6 +2809,10 @@ components: description: The packet destination port example: 55423 type: integer + error: + description: An optional error message if the packet failed to decode + example: Unable to decode EthernetType 49320 + type: string flags: description: 'The optional packet flags. Ex: SYN PSH FIN' example: @@ -2059,6 +2869,24 @@ components: example: 64296 type: integer type: object + model.PendingToolApproval: + description: |- + The tool_use in this sub-session awaiting the user's approval, if any (a + trailing tool_use with no tool_result that did not itself spawn a sub-session). + Derived server-side so the client resumes by POSTing these exact values rather + than re-deriving which nested tool is pending. Only populated for sub-sessions + (see GetSessionDetails); the root session's pending tool is re-derived + client-side from its trailing message. + properties: + input: + type: object + sessionId: + type: string + toolName: + type: string + toolUseId: + type: string + type: object model.Playbook: properties: contributors: @@ -2194,14 +3022,18 @@ components: can help determine if this is a false positive or a real attack. type: string fields: - description: not returned by the API, only used internally + description: Fields the converted query returns; used as result columns. items: type: string type: array uniqueItems: false filledQuery: - description: The query after variable substitution has been performed. + description: The query after LEGACY {field} variable substitution (queryVariableSubstitution) + has been performed. type: string + isAggregate: + description: Whether the query is a Sigma aggregation. + type: boolean oqlQuery: description: The event-specific query in OQL format type: string @@ -2217,6 +3049,9 @@ components: $ref: '#/components/schemas/model.EventRecord' type: array uniqueItems: false + queryTimedOut: + description: Indicates if the QueryResults were cut short by a timeout. + type: boolean question: description: The human readable question to be asked. example: What is the source IP address of the alert? @@ -2277,6 +3112,11 @@ components: model.SessionUsage: description: Usage statistics for the session. properties: + modelUsage: + additionalProperties: + $ref: '#/components/schemas/model.ModelUsageStats' + description: Usage statistics per model@adapter combination. + type: object totalCredits: description: The total credits used during the session. example: 5 @@ -2301,6 +3141,14 @@ components: when the 'Show advanced settings' option is enabled in the user interface. example: false type: boolean + allowedNodeTypes: + description: |- + AllowedNodeTypes is a list of node types that are allowed for per-node configuration. + When set and the setting supports per-node configuration, only nodes with matching roles are shown. + items: + type: string + type: array + uniqueItems: false default: description: (metadata) The default value for this configuration setting, if available @@ -2317,6 +3165,10 @@ components: example: Optional configuration parameters made available as defaults for all rules and alerters type: string + duplicatedFromId: + description: DuplicatedFromID is the setting ID this setting was duplicated + from, if any. + type: string duplicates: description: (metadata) Indicates whether this setting can be duplicated. Duplicating settings is a complex area that can lead to system or upgrade @@ -2342,8 +3194,14 @@ components: helpLink: description: (metadata) An HTML page, relative to the doc URL, that may assist the user in understanding the purpose of this setting. - example: elastalert.html + example: elastalert type: string + history: + description: History of changes to this setting. + items: + $ref: '#/components/schemas/model.AuditHistory' + type: array + uniqueItems: false id: description: The ID of the configuration setting. Each period represents a nested level. @@ -2368,6 +3226,11 @@ components: description: The node ID to which this setting's value applies example: chi-so-001_standalone type: string + note: + description: An optional note explaining the reason for this configuration + change. + example: Enabling this for the new site deployment + type: string optionSeparator: description: (metadata) For multi-select options, the separator string used to store them into a single string @@ -2378,6 +3241,8 @@ components: type: string type: array uniqueItems: false + origin: + $ref: '#/components/schemas/model.SettingOrigin' readonly: description: (metadata) Indicates whether this setting is read-only. Read-only settings should not be modified. @@ -2408,6 +3273,10 @@ components: while still permitting new values to be applied. example: true type: boolean + storage: + description: (metadata) The preferred storage location for this setting. + example: db,omitempty + type: string syntax: description: (metadata) An optional syntax designator for validating the given setting value. @@ -2436,6 +3305,17 @@ components: example: some custom value type: string type: object + model.SettingOrigin: + description: Origin indicates where the setting's current value was loaded from. + type: string + x-enum-comments: + SettingOriginDB: stored in Postgres + SettingOriginDefault: value not yet customised + SettingOriginYaml: stored in a pillar YAML file + x-enum-varnames: + - SettingOriginDefault + - SettingOriginDB + - SettingOriginYaml model.Severity: description: The severity classification of this detection enum: @@ -2468,6 +3348,32 @@ components: - SigLangSigma - SigLangSuricata - SigLangYara + model.Skill: + properties: + enabled: + description: A disabled skill grants nothing, but is still published so + it can be re-enabled. + example: true + type: boolean + isSystem: + example: true + type: boolean + name: + example: Threat Intel Lookup + type: string + personaAddendum: + description: Admin-authored guidance; exposed because an admin wrote it. + example: Always cite the source of an indicator. + type: string + tools: + example: + - query_events + - query_cases + items: + type: string + type: array + uniqueItems: false + type: object model.Status: properties: alerts: @@ -2481,16 +3387,56 @@ components: example: my_subgrid_a type: string type: object - model.StoredMessage: - description: A stored message in the chat session. This contains metadata about - the message and its context not necessary for the conversation with the Assistant. + model.StoredAgent: properties: - createTime: - description: The date and time that this object was created. This is a read-only - field. - example: "2024-11-14T15:03:22Z" - type: string - id: + allowedSkills: + example: + - Hunt + - Respond + items: + type: string + type: array + uniqueItems: false + canDelegateTo: + example: + - Log Analyst + items: + type: string + type: array + uniqueItems: false + description: + example: Analyzes suspicious binaries and scripts + type: string + enabled: + description: Pointer so an absent field means enabled rather than disabled. + example: true + type: boolean + isOrchestrator: + example: false + type: boolean + model: + description: A model selector ("id@adapter" or bare id), not a display name. + example: claude-sonnet-4.5@SOAI + type: string + name: + example: Malware Analyst + type: string + persona: + description: Addendum to the built-in prompt for a system agent; the whole + prompt otherwise. + example: Prefer static analysis before detonating a sample. + type: string + type: object + model.StoredMessage: + description: A stored message in the chat session. This contains metadata about + the message and its context not necessary for the conversation with the Assistant. + properties: + createTime: + description: The date and time that this object was created. This is a read-only + field. + example: "2024-11-14T15:03:22Z" + type: string + id: description: The ID assigned to this object by the server. This is a read-only field. example: PdFc-JIBLkNJ8-bDfz47 @@ -2501,6 +3447,10 @@ components: type: string message: $ref: '#/components/schemas/model.Message' + model: + description: The model used for this message. + example: sonnet-4.5@SOAI + type: string operation: description: The operation that was applied to the object. This is a read-only field. @@ -2529,6 +3479,31 @@ components: example: socl_my_new_client type: string type: object + model.StoredSkill: + properties: + enabled: + description: Pointer so an absent field means enabled rather than disabled. + example: true + type: boolean + name: + example: Threat Intel Lookup + type: string + persona: + description: Addendum to the built-in guidance for a system skill; all of + it otherwise. + example: Always cite the source of an indicator. + type: string + tools: + description: Ignored for a system skill, whose tool set is fixed by the + built-in. + example: + - query_events + - query_cases + items: + type: string + type: array + uniqueItems: false + type: object model.Subgrid: properties: caCertificate: @@ -2588,6 +3563,30 @@ components: type: string type: object model.ToolRequest: + properties: + auxData: + description: Auxiliary data for certain tools. + type: object + model: + description: The model to use for this tool execution. + example: claude-sonnet-4.5 + type: string + params: + description: The parameters for this tool use. + type: object + rejected: + description: |- + Rejected indicates the user declined this tool: it is not executed; an error + tool_result is recorded instead so the turn (and any parallel siblings) resolves. + type: boolean + sessionId: + description: The sessionId this chat message belongs to. + example: chat_1757086398900_ykhmndscn + type: string + toolUseId: + description: The unique identifier for this tool use. + example: tooluse_mT45or7ISwSEUivo63nqow + type: string type: object model.UiElement: properties: @@ -2639,6 +3638,87 @@ components: user before it can be saved type: boolean type: object + model.UnassignedShardGroup: + properties: + canAllocate: + description: The datastore's verdict on whether the sampled shard can currently + be allocated. + example: "no" + type: string + count: + description: The number of shards in this group. + example: 2 + type: integer + deciders: + description: The allocation rules that rejected or throttled the sampled + shard. + items: + $ref: '#/components/schemas/model.AllocationDecider' + type: array + uniqueItems: false + failureDetails: + description: Present when the sampled shard became unassigned due to an + allocation failure. + example: 'failed shard on node [abc]: shard failure, reason [corrupt file]' + type: string + primary: + description: Whether this group contains primary shards rather than replicas. + example: true + type: boolean + reason: + description: The datastore's reason code for why these shards became unassigned. + example: NODE_LEFT + type: string + sampleIndex: + description: The index of the sampled shard. + example: so-logs-2026.07.21 + type: string + sampleShard: + description: The number of the sampled shard within its index. + example: 0 + type: integer + sampleStatus: + description: Whether the sampled shard was explained, capped before being + explained, or failed to be explained. + example: explained + type: string + since: + description: The date and time that the sampled shard became unassigned. + example: "2026-07-21T20:48:02.943Z" + type: string + type: object + model.UnassignedShards: + description: A summary of the shards that the datastore has been unable to assign + to a node. + properties: + groups: + description: The unassigned shards grouped by reason, primaries first. + items: + $ref: '#/components/schemas/model.UnassignedShardGroup' + type: array + uniqueItems: false + primaries: + description: The number of unassigned shards that are primaries. + example: 2 + type: integer + replicas: + description: The number of unassigned shards that are replicas. + example: 12 + type: integer + total: + description: The total number of unassigned shards. + example: 14 + type: integer + type: object + model.UpdateSessionRequest: + properties: + action: + example: add + type: string + tag: + example: shared + type: string + type: object model.Usage: description: Usage statistics after deducting the costs of the message. properties: @@ -2732,6 +3812,11 @@ components: type: object model.UserUsage: properties: + modelUsage: + additionalProperties: + $ref: '#/components/schemas/model.ModelUsageStats' + description: Usage statistics per model@adapter combination. + type: object totalCredits: description: The total credits used by the user in the date range. example: 5 @@ -2771,7 +3856,7 @@ components: server.AccessTokenResponse: properties: access_token: - description: The access token to be used for all other Connect API requests." + description: The access token to be used for all other API requests." example: ory_at_arkgYuJXYp5zwU8Xyh8-URW6QIUbaZVf4JwDPoNZh0g.YcF4W5i2qoQ2RTWZvLYLNIeUjGaUhYuewz9Gua0y7YA type: string expires_in: @@ -2819,6 +3904,8 @@ components: query: example: 'somefield: somevalue AND anotherfield: 123' type: string + useEsql: + type: boolean type: object server.GenPublicIdResp: properties: @@ -2826,6 +3913,23 @@ components: example: fb58abf3-0a6d-49af-b1a0-1eeabec07716 type: string type: object + server.ToggleDismissRequest: + description: ToggleDismissRequest specifies the desired dismissal state for + a notification. + properties: + isDismissed: + description: Indicates whether the notification should be dismissed. + example: true + type: boolean + type: object + server.ToggleReadRequest: + description: ToggleReadRequest specifies the desired read state for a notification. + properties: + isRead: + description: Indicates whether the notification should be marked as read. + example: true + type: boolean + type: object securitySchemes: basic: scheme: basic @@ -2842,7 +3946,7 @@ components: type: oauth2 externalDocs: description: Security Onion Documentation Online - url: https://docs.securityonion.net + url: https://securityonion.net/docs info: contact: name: Community Support @@ -2853,11 +3957,11 @@ info: name: Elastic 2.0 url: https://securityonion.net/terms/ termsOfService: https://securityonion.net/terms/ - title: Security Onion Connect API - version: 2.4.200 + title: Security Onion API + version: "" openapi: 3.1.0 paths: - /api/assistant/admin/{userId}/sessions: + /connect/assistant/admin/{userId}/sessions: get: description: Get chat session metadata from a specific user. parameters: @@ -2885,10 +3989,10 @@ paths: security: - bearer: - assistant/read_all - summary: Get Assistant Sessions + summary: Get Assistant Sessions For a Specific User tags: - Assistant - /api/assistant/admin/{userId}/sessions/{sessionId}/history: + /connect/assistant/admin/{userId}/sessions/{sessionId}/history: get: description: Retrieve a chat session's messages for a given user and session. parameters: @@ -2926,7 +4030,7 @@ paths: summary: Get Assistant Session History tags: - Assistant - /api/assistant/admin/sessions: + /connect/assistant/admin/sessions: get: description: Get a list of all previous chat session metadata across all users. responses: @@ -2947,10 +4051,10 @@ paths: security: - bearer: - assistant/read_all - summary: Get Assistant Sessions + summary: Get Assistant Sessions Across Users tags: - Assistant - /api/assistant/admin/stats: + /connect/assistant/admin/stats: get: description: Retrieve usage statistics of the AI assistant over a specified date range for every user. @@ -3000,9 +4104,87 @@ paths: summary: Get Usage Statistics tags: - Assistant - /api/assistant/balance: + /connect/assistant/agents/{name}: + delete: + description: Remove an admin-created agent. System agents can only be disabled. + parameters: + - description: Agent name + in: path + name: name + required: true + schema: + type: string + responses: + "200": + content: + application/json: + schema: + type: object + description: Agent deleted + "401": + description: Request was not properly authenticated + "403": + description: Insufficient permissions, or the agent is system-provided + "404": + description: Agent not found + "500": + description: Internal SOC error; review SOC logs + security: + - bearer: + - config/write + summary: Delete an Assistant Agent + tags: + - Assistant + put: + description: Create or update a single agent definition. The server merges it + into the stored set, so a stale client cannot revert other agents. + parameters: + - description: Agent name + in: path + name: name + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/model.StoredAgent' + description: Agent definition + required: true + responses: + "200": + content: + application/json: + schema: + type: object + description: Agent saved + "400": + description: The request body is invalid + "401": + description: Request was not properly authenticated + "403": + description: Insufficient permissions for this request + "500": + description: Internal SOC error; review SOC logs + security: + - bearer: + - config/write + summary: Save an Assistant Agent + tags: + - Assistant + /connect/assistant/balance/{modelAndAdapter}: get: description: Retrieve the current balance/usage information for the AI assistant. + parameters: + - description: 'Model selector to get balance for: the id@adapter pair (bare + id allowed, slashes in ids supported), or an agent name in agentic mode' + example: sonnet-4.5@SOAI + in: path + name: modelAndAdapter + required: true + schema: + type: string responses: "200": content: @@ -3024,10 +4206,22 @@ paths: summary: Get Assistant Balance tags: - Assistant - /api/assistant/chat: + /connect/assistant/chat: post: description: Send a message to the AI assistant and receive a response. Supports both streaming (SSE) and non-streaming responses. + parameters: + - description: Type of entity associated with this session (e.g., 'alert_investigation') + in: query + name: entityType + schema: + type: string + - description: ID of the entity associated with this session (e.g., alert's + soc_id) + in: query + name: entityId + schema: + type: string requestBody: content: application/json: @@ -3066,7 +4260,179 @@ paths: summary: Send Chat Message tags: - Assistant - /api/assistant/sessions: + /connect/assistant/memories: + get: + description: Retrieve a page of the memories the requestor is allowed to read. + A query orders results by semantic similarity instead of recency. + parameters: + - description: self, global, or all (default) + example: global + in: query + name: scope + schema: + type: string + - description: Narrow to one user; another user requires memory/read_all + example: 8beae4b5-275b-4669-b678-8cff894911b5 + in: query + name: userId + schema: + type: string + - description: Order results by similarity to this text + example: preferred timezone + in: query + name: q + schema: + type: string + - description: Page size + example: 25 + in: query + name: limit + schema: + type: integer + - description: Page offset + example: 0 + in: query + name: offset + schema: + type: integer + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/model.MemoryResults' + description: A page of memories + "401": + description: Request was not properly authenticated + "403": + description: Insufficient permissions for this request + "500": + description: Internal SOC error; review SOC logs + security: + - bearer: + - memory/read_authored + - bearer: + - memory/read_global + - bearer: + - memory/read_all + summary: List Assistant Memories + tags: + - Assistant + post: + description: Store a new user-defined memory. User-defined memories are never + rewritten or removed by the memory scanner. + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/model.MemoryRequest' + description: Memory to create + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/model.MemoryRecord' + description: The created memory + "400": + description: The request body is invalid + "401": + description: Request was not properly authenticated + "403": + description: Insufficient permissions for this request + "500": + description: Internal SOC error; review SOC logs + security: + - bearer: + - memory/write_self + - bearer: + - memory/write_global + summary: Create an Assistant Memory + tags: + - Assistant + /connect/assistant/memories/{id}: + delete: + description: Permanently remove a memory. + parameters: + - description: Memory ID + example: c3d44fb8-3bc2-46e2-a7d2-8a8983556d1a + in: path + name: id + required: true + schema: + type: string + responses: + "200": + content: + application/json: + schema: + type: object + description: Memory deleted + "401": + description: Request was not properly authenticated + "403": + description: Insufficient permissions for this request + "404": + description: Memory not found + "500": + description: Internal SOC error; review SOC logs + security: + - bearer: + - memory/write_self + - bearer: + - memory/write_global + - bearer: + - memory/write_all + summary: Delete an Assistant Memory + tags: + - Assistant + put: + description: Replace the text or scope of an existing memory, marking it user-defined + so the scanner leaves it alone. + parameters: + - description: Memory ID + example: c3d44fb8-3bc2-46e2-a7d2-8a8983556d1a + in: path + name: id + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/model.MemoryRequest' + description: Replacement memory + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/model.MemoryRecord' + description: The updated memory + "400": + description: The request body is invalid + "401": + description: Request was not properly authenticated + "403": + description: Insufficient permissions for this request + "404": + description: Memory not found + "500": + description: Internal SOC error; review SOC logs + security: + - bearer: + - memory/write_self + - bearer: + - memory/write_global + - bearer: + - memory/write_all + summary: Update an Assistant Memory + tags: + - Assistant + /connect/assistant/sessions: get: description: Retrieve a list of all previous chat session metadata for the authenticated user. @@ -3088,10 +4454,10 @@ paths: security: - bearer: - assistant/read_authored - summary: Get Assistant Sessions + summary: Get Your Assistant Sessions tags: - Assistant - /api/assistant/sessions/{sessionId}: + /connect/assistant/sessions/{sessionId}: delete: description: Delete a specific chat session and all its associated messages. parameters: @@ -3102,42 +4468,151 @@ paths: schema: type: string responses: - "204": - description: Session successfully deleted - "400": - description: The provided session ID is invalid or missing + "204": + description: Session successfully deleted + "400": + description: The provided session ID is invalid or missing + "401": + description: Request was not properly authenticated + "403": + description: Insufficient permissions for this request + "500": + description: Internal SOC error; review SOC logs + security: + - bearer: + - assistant/delete_authored + summary: Delete Your Assistant Session + tags: + - Assistant + get: + description: Retrieve the complete chat history and usage for a specific session. + Can lookup deleted sessions. + parameters: + - description: Session ID to retrieve history for + in: path + name: sessionId + required: true + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/model.AssistantSessionDetails' + description: Complete chat history and session details + "400": + description: The provided session ID is invalid or missing + "401": + description: Request was not properly authenticated + "403": + description: Insufficient permissions for this request + "500": + description: Internal SOC error; review SOC logs + security: + - bearer: + - assistant/read_authored + - bearer: + - assistant/read_shared + summary: Get Session Details For Your Session + tags: + - Assistant + put: + description: Allow a user to modify their own session tags. + parameters: + - description: Session ID to modify + in: path + name: sessionId + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/model.UpdateSessionRequest' + description: Indicate what field we're doing what operation with. + required: true + responses: + "206": + content: + application/json: + schema: + type: object + description: No content + "400": + description: The provided session ID is invalid or missing + "401": + description: Request was not properly authenticated + "403": + description: Insufficient permissions for this request + "404": + description: Session not found + "500": + description: Internal SOC error; review SOC logs + security: + - bearer: + - assistant/write_authored + summary: Update metadata for a Session + tags: + - Assistant + /connect/assistant/skills/{name}: + delete: + description: Remove an admin-created skill. System skills can only be disabled. + parameters: + - description: Skill name + in: path + name: name + required: true + schema: + type: string + responses: + "200": + content: + application/json: + schema: + type: object + description: Skill deleted "401": description: Request was not properly authenticated "403": - description: Insufficient permissions for this request + description: Insufficient permissions, or the skill is system-provided + "404": + description: Skill not found "500": description: Internal SOC error; review SOC logs security: - bearer: - - assistant/delete_authored - summary: Delete Session + - config/write + summary: Delete an Assistant Skill tags: - Assistant - get: - description: Retrieve the complete chat history for a specific session. + put: + description: Create or update a single skill definition, merged into the stored + set by the server. parameters: - - description: Session ID to retrieve history for + - description: Skill name in: path - name: sessionId + name: name required: true schema: type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/model.StoredSkill' + description: Skill definition + required: true responses: "200": content: application/json: schema: - items: - $ref: '#/components/schemas/model.StoredMessage' - type: array - description: Complete chat history for the session + type: object + description: Skill saved "400": - description: The provided session ID is invalid or missing + description: The request body is invalid "401": description: Request was not properly authenticated "403": @@ -3146,11 +4621,11 @@ paths: description: Internal SOC error; review SOC logs security: - bearer: - - assistant/read_authored - summary: Get Session History + - config/write + summary: Save an Assistant Skill tags: - Assistant - /api/assistant/tool/{name}: + /connect/assistant/tool/{name}: post: description: Execute a tool on behalf of the assistant and continue the conversation with the result. Note that more permissions may be checked depending on the @@ -3188,6 +4663,10 @@ paths: description: Request was not properly authenticated "403": description: Insufficient permissions for this request + "404": + description: No pending tool request with this ID exists in the session + "409": + description: A tool for the indicated session is already running "500": description: Internal SOC error; review SOC logs security: @@ -4229,6 +5708,8 @@ paths: description: Insufficient permissions for this request "409": description: Public ID conflicts with existing detection + "423": + description: Rule synchronization is currently blocked "500": description: Internal SOC error; review SOC logs security: @@ -4282,6 +5763,8 @@ paths: description: Detection not found "409": description: Public ID conflicts with existing detection + "423": + description: Rule synchronization is currently blocked "500": description: Internal SOC error; review SOC logs security: @@ -4350,6 +5833,8 @@ paths: description: Insufficient permissions for this request "404": description: Detection not found + "423": + description: Sync operation is blocked "500": description: Internal SOC error; review SOC logs security: @@ -4969,6 +6454,34 @@ paths: summary: Acknowledge Alerts tags: - Query + /connect/events/health: + get: + description: |- + Retrieves a point-in-time diagnostic snapshot of the event datastore: overall status, health + indicators, node listing, and non-default settings. When shard availability is degraded, the + unassigned shards are inventoried and one sampled shard per group is explained. Diagnosed + problems are reported as findings on the indicator they affect, ranked by severity. + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/model.EventsHealth' + description: The events health snapshot + "401": + description: Request was not properly authenticated + "403": + description: Insufficient permissions for this request + "405": + description: The event module is not loaded on the server + "500": + description: Events health could not be retrieved; review SOC logs + security: + - bearer: + - nodes/read + summary: Get Events Health + tags: + - Query /connect/grid/: get: description: |- @@ -4996,6 +6509,75 @@ paths: summary: Get Grid Nodes tags: - Grid + /connect/grid/metrics: + get: + description: Retrieves historical time-series metrics for a specific node and + metric type over a given time range. + parameters: + - description: The optional node ID to retrieve metrics for + in: query + name: nodeId + schema: + type: string + - description: The optional container name to retrieve metrics for + in: query + name: container + schema: + type: string + - description: The name of the metric to retrieve (e.g., cpu, memory, load, + disk, net) + in: query + name: metric + required: true + schema: + type: string + - description: Date range, in the specified timezone + example: 2024/12/03 03:09:31 PM - 2024/12/04 03:09:31 PM + in: query + name: range + required: true + schema: + type: string + - description: Timezone of the date range + example: America/New_York + in: query + name: zone + required: true + schema: + type: string + - description: Date range date format + example: 2006/01/02 3:04:05 PM + in: query + name: format + required: true + schema: + type: string + responses: + "200": + content: + application/json: + schema: + additionalProperties: + items: + $ref: '#/components/schemas/model.MetricSample' + type: array + type: object + description: A map where keys are metric names/sources and values are arrays + of metric samples + "400": + description: The provided input parameters are malformed or invalid + "401": + description: Request was not properly authenticated or authorized + "403": + description: Insufficient permissions for this request + "500": + description: Internal SOC error; review SOC logs + security: + - bearer: + - nodes/read + summary: Get Historical Grid Metrics + tags: + - Grid /connect/grid/status: get: description: |- @@ -5423,6 +7005,158 @@ paths: summary: Node Check-in tags: - Grid + /connect/notifications: + get: + description: Retrieves notifications for the current user, optionally filtered. + parameters: + - description: Filter parameter (all, unread, dismissed) + in: query + name: filter + schema: + type: string + responses: + "200": + content: + application/json: + schema: + items: + $ref: '#/components/schemas/model.NotificationRecord' + type: array + description: The list of notifications + "400": + description: License is invalid + "401": + description: Request was not properly authenticated + "403": + description: Insufficient permissions for this request + "405": + description: Notification module has not been enabled on the server + "500": + description: Internal SOC error; review SOC logs + security: + - bearer: + - notifications/read + summary: Get Notifications + tags: + - Notifications + /connect/notifications/{id}/audit: + get: + description: Retrieves team user state logs (view/dismiss) for a specific notification. + parameters: + - description: Notification ID + example: notif-1 + in: path + name: id + required: true + schema: + type: string + responses: + "200": + content: + application/json: + schema: + items: + $ref: '#/components/schemas/model.NotificationAuditEntry' + type: array + description: The list of notification user audit records + "400": + description: Invalid parameters + "401": + description: Request was not properly authenticated + "403": + description: Insufficient permissions for this request + "405": + description: Notification module has not been enabled on the server + "500": + description: Internal SOC error; review SOC logs + security: + - bearer: + - notifications/read_all + summary: Get Notification Audit Logs + tags: + - Notifications + /connect/notifications/{id}/dismiss: + put: + description: Marks a notification as dismissed or active for the current user. + parameters: + - description: Notification ID + example: notif-1 + in: path + name: id + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/server.ToggleDismissRequest' + description: Payload to toggle dismiss + required: true + responses: + "200": + content: + application/json: + schema: + type: object + description: The notification dismissal state was successfully updated + "400": + description: Invalid request body or parameters + "401": + description: Request was not properly authenticated + "403": + description: Insufficient permissions for this request + "405": + description: Notification module has not been enabled on the server + "500": + description: Internal SOC error; review SOC logs + security: + - bearer: + - notifications/write + summary: Toggle Dismiss Notification + tags: + - Notifications + /connect/notifications/{id}/read: + put: + description: Marks a notification as read or unread for the current user. + parameters: + - description: Notification ID + example: notif-1 + in: path + name: id + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/server.ToggleReadRequest' + description: Payload to toggle read + required: true + responses: + "200": + content: + application/json: + schema: + type: object + description: The notification read state was successfully updated + "400": + description: Invalid request body or parameters + "401": + description: Request was not properly authenticated + "403": + description: Insufficient permissions for this request + "405": + description: Notification module has not been enabled on the server + "500": + description: Internal SOC error; review SOC logs + security: + - bearer: + - notifications/write + summary: Toggle Read Notification + tags: + - Notifications /connect/packets/{jobId}: get: description: Retrieves the packets collected and attached to the job represented @@ -5442,6 +7176,13 @@ paths: name: unwrap schema: type: boolean + - description: If true, packet decode failures will be excluded from the returned + results. + example: true + in: query + name: excludeErrors + schema: + type: boolean - description: The starting offset of the packet to retrieve; used for paging large packet results. Defaults to 0. example: 100 @@ -5466,6 +7207,12 @@ paths: $ref: '#/components/schemas/model.Packet' type: array description: The array of retrieved Packet objects + headers: + X-Decode-Errors-Present: + description: Indicates whether any packets in the PCAP capture encountered + decode failures (true or false) + schema: + type: string "400": description: The provided input object or parameters are malformed or invalid "401": @@ -5562,6 +7309,22 @@ paths: required: true schema: type: string + - description: 'How much of the query pipeline to run: skeleton (questions only), + convert (questions with OQL queries), or full (queries executed; the default)' + in: query + name: stage + schema: + enum: + - skeleton + - convert + - full + type: string + - description: Alert timestamp (RFC3339 or '2006-01-02 15:04:05') used to narrow + the alert lookup; ignored if absent or unparseable + in: query + name: ts + schema: + type: string responses: "200": content: @@ -5587,6 +7350,49 @@ paths: summary: Get Event-Specific Playbook tags: - Playbooks + /connect/playbook/question: + post: + description: 'Executes a single converted playbook question and returns it with + queryResults populated. A question without a range is rejected: it is answered + by the alert itself, not this route.' + parameters: + - description: Alert timestamp that anchors the question's relative range (RFC3339 + or '2006-01-02 15:04:05') + in: query + name: ts + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/model.Question' + description: The question to execute; needs the range and converted oqlQuery, + plus fields and isAggregate from conversion + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/model.Question' + description: The question was executed + "400": + description: Malformed request + "401": + description: Request was not properly authenticated + "403": + description: Insufficient permissions for this request + "500": + description: Internal SOC error; review SOC logs + security: + - bearer: + - playbooks/read + - events/read + summary: Execute Playbook Question + tags: + - Playbooks /connect/query/{operation}: get: description: Requests the server provide a new query for the SOC client to show, @@ -6193,7 +7999,7 @@ paths: /oauth2/token: post: description: |- - Exchanges a client ID and client secret for a temporary access token needed for calling Security Onion Connect API methods. + Exchanges a client ID and client secret for a temporary access token needed for calling Security Onion API methods. The client ID and client secret are provided within the SOC Administration -> API Clients screen, when creating a new API client or when regenerating an API client's secret. The client secrets are only temporarily visible during those two specific times. The returned access token will expire within 1-2 hours, by default. Ensure the custom integration application is capable of exchanging for a new access token prior to the expiration. @@ -6224,3 +8030,5 @@ paths: summary: Obtain Access Token tags: - Authentication +servers: +- url: https://BASE_URL