From b74f0b04de5b267e43cbf6a88a2fdd7e731c9cf5 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 16 Feb 2024 11:18:20 -0700 Subject: [PATCH 01/79] bump version for v24.02.1 --- docker-compose-dev.yml | 46 +++++----- docker-compose.yml | 46 +++++----- docs/api-version.md | 2 +- docs/contributing-pcap.md | 2 +- docs/download.md | 6 +- docs/hedgehog-iso-build.md | 2 +- docs/kubernetes.md | 88 +++++++++---------- docs/malcolm-iso.md | 2 +- docs/quickstart.md | 38 ++++---- docs/ubuntu-install-example.md | 38 ++++---- kubernetes/03-opensearch.yml | 4 +- kubernetes/04-dashboards.yml | 2 +- kubernetes/05-upload.yml | 4 +- kubernetes/06-pcap-monitor.yml | 4 +- kubernetes/07-arkime.yml | 4 +- kubernetes/08-api.yml | 2 +- kubernetes/09-dashboards-helper.yml | 2 +- kubernetes/10-zeek.yml | 4 +- kubernetes/11-suricata.yml | 4 +- kubernetes/12-file-monitor.yml | 4 +- kubernetes/13-filebeat.yml | 4 +- kubernetes/14-logstash.yml | 4 +- kubernetes/15-netbox-redis.yml | 4 +- kubernetes/16-netbox-redis-cache.yml | 2 +- kubernetes/17-netbox-postgres.yml | 4 +- kubernetes/18-netbox.yml | 4 +- kubernetes/19-htadmin.yml | 4 +- kubernetes/20-pcap-capture.yml | 4 +- kubernetes/21-zeek-live.yml | 4 +- kubernetes/22-suricata-live.yml | 4 +- kubernetes/23-arkime-live.yml | 4 +- kubernetes/24-freq.yml | 2 +- kubernetes/98-nginx-proxy.yml | 4 +- .../aws/ami/packer_vars.json.example | 2 +- 34 files changed, 177 insertions(+), 177 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 0639b407a..313cce697 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -15,7 +15,7 @@ services: build: context: . dockerfile: Dockerfiles/opensearch.Dockerfile - image: ghcr.io/idaholab/malcolm/opensearch:24.02.0 + image: ghcr.io/idaholab/malcolm/opensearch:24.02.1 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -60,7 +60,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards-helper.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.02.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.02.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -93,7 +93,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards:24.02.0 + image: ghcr.io/idaholab/malcolm/dashboards:24.02.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -124,7 +124,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: ghcr.io/idaholab/malcolm/logstash-oss:24.02.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.02.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -178,7 +178,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.02.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -216,7 +216,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:24.02.0 + image: ghcr.io/idaholab/malcolm/arkime:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -255,7 +255,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:24.02.0 + image: ghcr.io/idaholab/malcolm/arkime:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -294,7 +294,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:24.02.0 + image: ghcr.io/idaholab/malcolm/zeek:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -330,7 +330,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:24.02.0 + image: ghcr.io/idaholab/malcolm/zeek:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -362,7 +362,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:24.02.0 + image: ghcr.io/idaholab/malcolm/suricata:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -393,7 +393,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:24.02.0 + image: ghcr.io/idaholab/malcolm/suricata:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -426,7 +426,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/file-monitor:24.02.0 + image: ghcr.io/idaholab/malcolm/file-monitor:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -458,7 +458,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-capture:24.02.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -490,7 +490,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.02.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -521,7 +521,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: ghcr.io/idaholab/malcolm/file-upload:24.02.0 + image: ghcr.io/idaholab/malcolm/file-upload:24.02.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -548,7 +548,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:24.02.0 + image: ghcr.io/idaholab/malcolm/htadmin:24.02.1 profiles: ["malcolm"] logging: *default-logging build: @@ -578,7 +578,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:24.02.0 + image: ghcr.io/idaholab/malcolm/freq:24.02.1 profiles: ["malcolm"] logging: *default-logging build: @@ -605,7 +605,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:24.02.0 + image: ghcr.io/idaholab/malcolm/netbox:24.02.1 profiles: ["malcolm"] logging: *default-logging build: @@ -642,7 +642,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:24.02.0 + image: ghcr.io/idaholab/malcolm/postgresql:24.02.1 profiles: ["malcolm"] logging: *default-logging build: @@ -671,7 +671,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:24.02.0 + image: ghcr.io/idaholab/malcolm/redis:24.02.1 profiles: ["malcolm"] logging: *default-logging build: @@ -704,7 +704,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:24.02.0 + image: ghcr.io/idaholab/malcolm/redis:24.02.1 profiles: ["malcolm"] logging: *default-logging build: @@ -736,7 +736,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:24.02.0 + image: ghcr.io/idaholab/malcolm/api:24.02.1 profiles: ["malcolm"] logging: *default-logging build: @@ -769,7 +769,7 @@ services: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.02.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.02.1 profiles: ["malcolm"] logging: *default-logging restart: "no" diff --git a/docker-compose.yml b/docker-compose.yml index ae9444ecc..4cb47364f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ x-logging: services: opensearch: - image: ghcr.io/idaholab/malcolm/opensearch:24.02.0 + image: ghcr.io/idaholab/malcolm/opensearch:24.02.1 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -54,7 +54,7 @@ services: retries: 3 start_period: 180s dashboards-helper: - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.02.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.02.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -84,7 +84,7 @@ services: retries: 3 start_period: 30s dashboards: - image: ghcr.io/idaholab/malcolm/dashboards:24.02.0 + image: ghcr.io/idaholab/malcolm/dashboards:24.02.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -112,7 +112,7 @@ services: retries: 3 start_period: 210s logstash: - image: ghcr.io/idaholab/malcolm/logstash-oss:24.02.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.02.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -159,7 +159,7 @@ services: retries: 3 start_period: 600s filebeat: - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.02.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -194,7 +194,7 @@ services: retries: 3 start_period: 60s arkime: - image: ghcr.io/idaholab/malcolm/arkime:24.02.0 + image: ghcr.io/idaholab/malcolm/arkime:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -228,7 +228,7 @@ services: retries: 3 start_period: 210s arkime-live: - image: ghcr.io/idaholab/malcolm/arkime:24.02.0 + image: ghcr.io/idaholab/malcolm/arkime:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -261,7 +261,7 @@ services: - ./arkime/rules:/opt/arkime/rules:ro - ./pcap:/data/pcap zeek: - image: ghcr.io/idaholab/malcolm/zeek:24.02.0 + image: ghcr.io/idaholab/malcolm/zeek:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -293,7 +293,7 @@ services: retries: 3 start_period: 60s zeek-live: - image: ghcr.io/idaholab/malcolm/zeek:24.02.0 + image: ghcr.io/idaholab/malcolm/zeek:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -321,7 +321,7 @@ services: - ./zeek/intel:/opt/zeek/share/zeek/site/intel - ./zeek/custom:/opt/zeek/share/zeek/site/custom:ro suricata: - image: ghcr.io/idaholab/malcolm/suricata:24.02.0 + image: ghcr.io/idaholab/malcolm/suricata:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -349,7 +349,7 @@ services: retries: 3 start_period: 120s suricata-live: - image: ghcr.io/idaholab/malcolm/suricata:24.02.0 + image: ghcr.io/idaholab/malcolm/suricata:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -379,7 +379,7 @@ services: - ./suricata/rules:/opt/suricata/rules:ro - ./suricata/include-configs:/opt/suricata/include-configs:ro file-monitor: - image: ghcr.io/idaholab/malcolm/file-monitor:24.02.0 + image: ghcr.io/idaholab/malcolm/file-monitor:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -408,7 +408,7 @@ services: retries: 3 start_period: 60s pcap-capture: - image: ghcr.io/idaholab/malcolm/pcap-capture:24.02.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -437,7 +437,7 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./pcap/upload:/pcap pcap-monitor: - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.02.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.02.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -465,7 +465,7 @@ services: retries: 3 start_period: 90s upload: - image: ghcr.io/idaholab/malcolm/file-upload:24.02.0 + image: ghcr.io/idaholab/malcolm/file-upload:24.02.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -492,7 +492,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:24.02.0 + image: ghcr.io/idaholab/malcolm/htadmin:24.02.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -519,7 +519,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:24.02.0 + image: ghcr.io/idaholab/malcolm/freq:24.02.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -543,7 +543,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:24.02.0 + image: ghcr.io/idaholab/malcolm/netbox:24.02.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -577,7 +577,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:24.02.0 + image: ghcr.io/idaholab/malcolm/postgresql:24.02.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -603,7 +603,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:24.02.0 + image: ghcr.io/idaholab/malcolm/redis:24.02.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -633,7 +633,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:24.02.0 + image: ghcr.io/idaholab/malcolm/redis:24.02.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -662,7 +662,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:24.02.0 + image: ghcr.io/idaholab/malcolm/api:24.02.1 profiles: ["malcolm"] logging: *default-logging command: gunicorn --bind 0:5000 manage:app @@ -689,7 +689,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.02.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.02.1 profiles: ["malcolm"] logging: *default-logging restart: "no" diff --git a/docs/api-version.md b/docs/api-version.md index 76364f95c..e281fba1a 100644 --- a/docs/api-version.md +++ b/docs/api-version.md @@ -47,6 +47,6 @@ Returns version information about Malcolm and version/[health](https://opensearc } }, "sha": "77574975", - "version": "24.02.0" + "version": "24.02.1" } ``` diff --git a/docs/contributing-pcap.md b/docs/contributing-pcap.md index 167235639..df4961b76 100644 --- a/docs/contributing-pcap.md +++ b/docs/contributing-pcap.md @@ -1,6 +1,6 @@ # PCAP processors -When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v24.02.0 release]({{ site.github.repository_url }}/releases/tag/v24.02.0)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: +When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v24.02.1 release]({{ site.github.repository_url }}/releases/tag/v24.02.1)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: 1. Define the service as instructed in the [Adding a new service](contributing-new-image.md#NewImage) section * Note how the existing `zeek` and `arkime` services use [bind mounts](contributing-local-modifications.md#Bind) to access the local `./pcap` directory diff --git a/docs/download.md b/docs/download.md index 083f84936..e7ad1cbe3 100644 --- a/docs/download.md +++ b/docs/download.md @@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [malcolm-24.02.0.iso](/iso/malcolm-24.02.0.iso) (5.1GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/malcolm-24.02.0.iso.sha256.txt) | +| [malcolm-24.02.1.iso](/iso/malcolm-24.02.1.iso) (5.1GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/malcolm-24.02.1.iso.sha256.txt) | ## Hedgehog Linux @@ -26,7 +26,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [hedgehog-24.02.0.iso](/iso/hedgehog-24.02.0.iso) (2.5GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/hedgehog-24.02.0.iso.sha256.txt) | +| [hedgehog-24.02.1.iso](/iso/hedgehog-24.02.1.iso) (2.5GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/hedgehog-24.02.1.iso.sha256.txt) | ### Raspberry Pi 4 Image @@ -34,7 +34,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | Image | SHA256 | |---|---| -| [hedgehog-24.02.0_raspi_4.img.xz](/iso/hedgehog-24.02.0_raspi_4.img.xz) (1.4GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/hedgehog-24.02.0_raspi_4.img.xz.sha256.txt) | +| [hedgehog-24.02.1_raspi_4.img.xz](/iso/hedgehog-24.02.1_raspi_4.img.xz) (1.4GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/hedgehog-24.02.1_raspi_4.img.xz.sha256.txt) | ## Warning diff --git a/docs/hedgehog-iso-build.md b/docs/hedgehog-iso-build.md index 1c4e7faa0..e7a700f43 100644 --- a/docs/hedgehog-iso-build.md +++ b/docs/hedgehog-iso-build.md @@ -29,7 +29,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-24.02.0.iso" +Finished, created "/sensor-build/hedgehog-24.02.1.iso" … ``` diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 3b41df9a5..8075932a7 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -272,28 +272,28 @@ agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | 861.34m | 14.36% | 19.55Gi | 9.29Gi | 61.28Gi | 11 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:24.02.0 | -file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:24.02.0 | -zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:24.02.0 | -dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:24.02.0 | -upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:24.02.0 | -filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:24.02.0 | -zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:24.02.0 | -logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:24.02.0 | -netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:24.02.0 | -suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:24.02.0 | -dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:24.02.0 | -netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:24.02.0 | -suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:24.02.0 | -freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:24.02.0 | -arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:24.02.0 | -pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:24.02.0 | -pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:24.02.0 | -netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:24.02.0 | -htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:24.02.0 | -netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:24.02.0 | -nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:24.02.0 | -opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:24.02.0 | +api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:24.02.1 | +file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:24.02.1 | +zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:24.02.1 | +dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:24.02.1 | +upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:24.02.1 | +filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:24.02.1 | +zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:24.02.1 | +logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:24.02.1 | +netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:24.02.1 | +suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:24.02.1 | +dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:24.02.1 | +netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:24.02.1 | +suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:24.02.1 | +freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:24.02.1 | +arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:24.02.1 | +pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:24.02.1 | +pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:24.02.1 | +netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:24.02.1 | +htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:24.02.1 | +netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:24.02.1 | +nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:24.02.1 | +opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:24.02.1 | ``` The other control scripts (`stop`, `restart`, `logs`, etc.) work in a similar manner as in a Docker-based deployment. One notable difference is the `wipe` script: data on PersistentVolume storage cannot be deleted by `wipe`. It must be deleted manually on the storage media underlying the PersistentVolumes. @@ -553,28 +553,28 @@ agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | 552.71m | 9.21% | 19.55Gi | 13.27Gi | 61.28Gi | 12 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:24.02.0 | -netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:24.02.0 | -dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:24.02.0 | -freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:24.02.0 | -pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:24.02.0 | -nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:24.02.0 | -htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:24.02.0 | -opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:24.02.0 | -zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:24.02.0 | -dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:24.02.0 | -arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:24.02.0 | -api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:24.02.0 | -netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:24.02.0 | -pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:24.02.0 | -suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:24.02.0 | -suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:24.02.0 | -netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:24.02.0 | -zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:24.02.0 | -filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:24.02.0 | -file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:24.02.0 | -upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:24.02.0 | -logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:24.02.0 | +netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:24.02.1 | +netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:24.02.1 | +dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:24.02.1 | +freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:24.02.1 | +pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:24.02.1 | +nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:24.02.1 | +htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:24.02.1 | +opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:24.02.1 | +zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:24.02.1 | +dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:24.02.1 | +arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:24.02.1 | +api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:24.02.1 | +netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:24.02.1 | +pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:24.02.1 | +suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:24.02.1 | +suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:24.02.1 | +netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:24.02.1 | +zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:24.02.1 | +filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:24.02.1 | +file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:24.02.1 | +upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:24.02.1 | +logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:24.02.1 | ``` View container logs for the Malcolm deployment with `./scripts/logs` (if **[stern](https://github.com/stern/stern)** present in `$PATH`): diff --git a/docs/malcolm-iso.md b/docs/malcolm-iso.md index da5e35fe0..d2e3e627e 100644 --- a/docs/malcolm-iso.md +++ b/docs/malcolm-iso.md @@ -41,7 +41,7 @@ Building the ISO may take 30 minutes or more depending on the system. As the bui ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-24.02.0.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-24.02.1.iso" … ``` diff --git a/docs/quickstart.md b/docs/quickstart.md index de66729d6..dc50aa7d9 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -54,25 +54,25 @@ You can then observe the images have been retrieved by running `docker images`: ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 24.02.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 24.02.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 24.02.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 24.02.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 24.02.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 24.02.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.02.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 24.02.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 24.02.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 24.02.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 24.02.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 24.02.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 24.02.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 24.02.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 24.02.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 24.02.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 24.02.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 24.02.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 24.02.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 24.02.1 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 24.02.1 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 24.02.1 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 24.02.1 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 24.02.1 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 24.02.1 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.02.1 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 24.02.1 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 24.02.1 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 24.02.1 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 24.02.1 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 24.02.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 24.02.1 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 24.02.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.02.1 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 24.02.1 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 24.02.1 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 24.02.1 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 24.02.1 xxxxxxxxxxxx 3 days ago 1GB ``` ### Import from pre-packaged tarballs diff --git a/docs/ubuntu-install-example.md b/docs/ubuntu-install-example.md index 179901b8f..69705bd29 100644 --- a/docs/ubuntu-install-example.md +++ b/docs/ubuntu-install-example.md @@ -257,25 +257,25 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 24.02.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 24.02.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 24.02.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 24.02.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 24.02.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 24.02.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.02.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 24.02.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 24.02.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 24.02.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 24.02.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 24.02.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 24.02.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 24.02.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 24.02.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 24.02.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 24.02.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 24.02.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 24.02.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 24.02.1 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 24.02.1 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 24.02.1 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 24.02.1 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 24.02.1 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 24.02.1 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.02.1 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 24.02.1 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 24.02.1 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 24.02.1 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 24.02.1 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 24.02.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 24.02.1 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 24.02.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.02.1 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 24.02.1 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 24.02.1 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 24.02.1 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 24.02.1 xxxxxxxxxxxx 3 days ago 1GB ``` Finally, start Malcolm. When Malcolm starts it will stream informational and debug messages to the console until it has completed initializing. diff --git a/kubernetes/03-opensearch.yml b/kubernetes/03-opensearch.yml index 8f82de56e..53f9baa5b 100644 --- a/kubernetes/03-opensearch.yml +++ b/kubernetes/03-opensearch.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: opensearch-container - image: ghcr.io/idaholab/malcolm/opensearch:24.02.0 + image: ghcr.io/idaholab/malcolm/opensearch:24.02.1 imagePullPolicy: Always stdin: false tty: true @@ -71,7 +71,7 @@ spec: subPath: "opensearch" initContainers: - name: opensearch-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/04-dashboards.yml b/kubernetes/04-dashboards.yml index dc9fe4aca..82b4d0e42 100644 --- a/kubernetes/04-dashboards.yml +++ b/kubernetes/04-dashboards.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-container - image: ghcr.io/idaholab/malcolm/dashboards:24.02.0 + image: ghcr.io/idaholab/malcolm/dashboards:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/05-upload.yml b/kubernetes/05-upload.yml index 54ac4699e..70616e5f0 100644 --- a/kubernetes/05-upload.yml +++ b/kubernetes/05-upload.yml @@ -34,7 +34,7 @@ spec: spec: containers: - name: upload-container - image: ghcr.io/idaholab/malcolm/file-upload:24.02.0 + image: ghcr.io/idaholab/malcolm/file-upload:24.02.1 imagePullPolicy: Always stdin: false tty: true @@ -73,7 +73,7 @@ spec: subPath: "upload" initContainers: - name: upload-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/06-pcap-monitor.yml b/kubernetes/06-pcap-monitor.yml index f3fe9213a..675597120 100644 --- a/kubernetes/06-pcap-monitor.yml +++ b/kubernetes/06-pcap-monitor.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: pcap-monitor-container - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.02.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.02.1 imagePullPolicy: Always stdin: false tty: true @@ -70,7 +70,7 @@ spec: name: pcap-monitor-zeek-volume initContainers: - name: pcap-monitor-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/07-arkime.yml b/kubernetes/07-arkime.yml index 9358a7980..c2984cc06 100644 --- a/kubernetes/07-arkime.yml +++ b/kubernetes/07-arkime.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: arkime-container - image: ghcr.io/idaholab/malcolm/arkime:24.02.0 + image: ghcr.io/idaholab/malcolm/arkime:24.02.1 imagePullPolicy: Always stdin: false tty: true @@ -79,7 +79,7 @@ spec: name: arkime-pcap-volume initContainers: - name: arkime-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/08-api.yml b/kubernetes/08-api.yml index cd8462b80..36f5d7a39 100644 --- a/kubernetes/08-api.yml +++ b/kubernetes/08-api.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: api-container - image: ghcr.io/idaholab/malcolm/api:24.02.0 + image: ghcr.io/idaholab/malcolm/api:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/09-dashboards-helper.yml b/kubernetes/09-dashboards-helper.yml index 4f8598940..b3223f374 100644 --- a/kubernetes/09-dashboards-helper.yml +++ b/kubernetes/09-dashboards-helper.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-helper-container - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.02.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index 34352453c..02e84f99b 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-offline-container - image: ghcr.io/idaholab/malcolm/zeek:24.02.0 + image: ghcr.io/idaholab/malcolm/zeek:24.02.1 imagePullPolicy: Always stdin: false tty: true @@ -64,7 +64,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/11-suricata.yml b/kubernetes/11-suricata.yml index c19818116..f838bd17b 100644 --- a/kubernetes/11-suricata.yml +++ b/kubernetes/11-suricata.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-offline-container - image: ghcr.io/idaholab/malcolm/suricata:24.02.0 + image: ghcr.io/idaholab/malcolm/suricata:24.02.1 imagePullPolicy: Always stdin: false tty: true @@ -55,7 +55,7 @@ spec: name: suricata-offline-custom-configs-volume initContainers: - name: suricata-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/12-file-monitor.yml b/kubernetes/12-file-monitor.yml index ed4ee049c..1bd47d08d 100644 --- a/kubernetes/12-file-monitor.yml +++ b/kubernetes/12-file-monitor.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: file-monitor-container - image: ghcr.io/idaholab/malcolm/file-monitor:24.02.0 + image: ghcr.io/idaholab/malcolm/file-monitor:24.02.1 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: name: file-monitor-yara-rules-custom-volume initContainers: - name: file-monitor-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/13-filebeat.yml b/kubernetes/13-filebeat.yml index 553f04d46..c14bbe34e 100644 --- a/kubernetes/13-filebeat.yml +++ b/kubernetes/13-filebeat.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: filebeat-container - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.02.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.02.1 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: "nginx" initContainers: - name: filebeat-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/14-logstash.yml b/kubernetes/14-logstash.yml index 47b4943a4..7239101bf 100644 --- a/kubernetes/14-logstash.yml +++ b/kubernetes/14-logstash.yml @@ -49,7 +49,7 @@ spec: # topologyKey: "kubernetes.io/hostname" containers: - name: logstash-container - image: ghcr.io/idaholab/malcolm/logstash-oss:24.02.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.02.1 imagePullPolicy: Always stdin: false tty: true @@ -115,7 +115,7 @@ spec: subPath: "logstash" initContainers: - name: logstash-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/15-netbox-redis.yml b/kubernetes/15-netbox-redis.yml index 27e54661d..a85550502 100644 --- a/kubernetes/15-netbox-redis.yml +++ b/kubernetes/15-netbox-redis.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-container - image: ghcr.io/idaholab/malcolm/redis:24.02.0 + image: ghcr.io/idaholab/malcolm/redis:24.02.1 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: netbox/redis initContainers: - name: netbox-redis-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/16-netbox-redis-cache.yml b/kubernetes/16-netbox-redis-cache.yml index 011f7baf7..e1aaa41b6 100644 --- a/kubernetes/16-netbox-redis-cache.yml +++ b/kubernetes/16-netbox-redis-cache.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-cache-container - image: ghcr.io/idaholab/malcolm/redis:24.02.0 + image: ghcr.io/idaholab/malcolm/redis:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/17-netbox-postgres.yml b/kubernetes/17-netbox-postgres.yml index 2b345240f..3f40e3cd6 100644 --- a/kubernetes/17-netbox-postgres.yml +++ b/kubernetes/17-netbox-postgres.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-postgres-container - image: ghcr.io/idaholab/malcolm/postgresql:24.02.0 + image: ghcr.io/idaholab/malcolm/postgresql:24.02.1 imagePullPolicy: Always stdin: false tty: true @@ -74,7 +74,7 @@ spec: subPath: netbox/postgres initContainers: - name: netbox-postgres-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index 37577f687..75475df24 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -36,7 +36,7 @@ spec: spec: containers: - name: netbox-container - image: ghcr.io/idaholab/malcolm/netbox:24.02.0 + image: ghcr.io/idaholab/malcolm/netbox:24.02.1 imagePullPolicy: Always stdin: false tty: true @@ -88,7 +88,7 @@ spec: subPath: netbox/media initContainers: - name: netbox-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/19-htadmin.yml b/kubernetes/19-htadmin.yml index f39cb01d7..c2bfaaea1 100644 --- a/kubernetes/19-htadmin.yml +++ b/kubernetes/19-htadmin.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: htadmin-container - image: ghcr.io/idaholab/malcolm/htadmin:24.02.0 + image: ghcr.io/idaholab/malcolm/htadmin:24.02.1 imagePullPolicy: Always stdin: false tty: true @@ -63,7 +63,7 @@ spec: subPath: "htadmin" initContainers: - name: htadmin-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index f0ddd2ff8..e430dd8fd 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: pcap-capture-container - image: ghcr.io/idaholab/malcolm/pcap-capture:24.02.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.02.1 imagePullPolicy: Always stdin: false tty: true @@ -50,7 +50,7 @@ spec: subPath: "upload" initContainers: - name: pcap-capture-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index 1af8ac204..67e7e736b 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-live-container - image: ghcr.io/idaholab/malcolm/zeek:24.02.0 + image: ghcr.io/idaholab/malcolm/zeek:24.02.1 imagePullPolicy: Always stdin: false tty: true @@ -61,7 +61,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index bd1ca6e31..17e2204be 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-live-container - image: ghcr.io/idaholab/malcolm/suricata:24.02.0 + image: ghcr.io/idaholab/malcolm/suricata:24.02.1 imagePullPolicy: Always stdin: false tty: true @@ -56,7 +56,7 @@ spec: name: suricata-live-custom-configs-volume initContainers: - name: suricata-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/23-arkime-live.yml b/kubernetes/23-arkime-live.yml index a84945410..0429c6ea8 100644 --- a/kubernetes/23-arkime-live.yml +++ b/kubernetes/23-arkime-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: arkime-live-container - image: ghcr.io/idaholab/malcolm/arkime:24.02.0 + image: ghcr.io/idaholab/malcolm/arkime:24.02.1 imagePullPolicy: Always stdin: false tty: true @@ -62,7 +62,7 @@ spec: name: arkime-live-pcap-volume initContainers: - name: arkime-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/24-freq.yml b/kubernetes/24-freq.yml index 6fcd077cd..9f158303e 100644 --- a/kubernetes/24-freq.yml +++ b/kubernetes/24-freq.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: freq-container - image: ghcr.io/idaholab/malcolm/freq:24.02.0 + image: ghcr.io/idaholab/malcolm/freq:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/98-nginx-proxy.yml b/kubernetes/98-nginx-proxy.yml index 35dbe4986..61847052e 100644 --- a/kubernetes/98-nginx-proxy.yml +++ b/kubernetes/98-nginx-proxy.yml @@ -39,7 +39,7 @@ spec: spec: containers: - name: nginx-proxy-container - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.02.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.02.1 imagePullPolicy: Always stdin: false tty: true @@ -99,7 +99,7 @@ spec: subPath: "nginx" initContainers: - name: nginx-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 imagePullPolicy: Always stdin: false tty: true diff --git a/scripts/third-party-environments/aws/ami/packer_vars.json.example b/scripts/third-party-environments/aws/ami/packer_vars.json.example index 511dfa2b8..59cc8544b 100644 --- a/scripts/third-party-environments/aws/ami/packer_vars.json.example +++ b/scripts/third-party-environments/aws/ami/packer_vars.json.example @@ -2,7 +2,7 @@ "aws_access_key": "XXXXXXXXXXXXXXXXXXXX", "aws_secret_key": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "instance_type": "t2.micro", - "malcolm_tag": "v24.02.0", + "malcolm_tag": "v24.02.1", "malcolm_repo": "idaholab/Malcolm", "malcolm_uid": "1000", "ssh_username": "ec2-user", From d9e20ded2a0af9bfc7c1ef112efd477b183256af Mon Sep 17 00:00:00 2001 From: SG Date: Fri, 16 Feb 2024 11:31:02 -0700 Subject: [PATCH 02/79] bump development for v24.03.0 --- docker-compose-dev.yml | 46 +++++----- docker-compose.yml | 46 +++++----- docs/api-version.md | 2 +- docs/contributing-pcap.md | 2 +- docs/download.md | 6 +- docs/hedgehog-iso-build.md | 2 +- docs/kubernetes.md | 88 +++++++++---------- docs/malcolm-iso.md | 2 +- docs/quickstart.md | 38 ++++---- docs/ubuntu-install-example.md | 38 ++++---- kubernetes/03-opensearch.yml | 4 +- kubernetes/04-dashboards.yml | 2 +- kubernetes/05-upload.yml | 4 +- kubernetes/06-pcap-monitor.yml | 4 +- kubernetes/07-arkime.yml | 4 +- kubernetes/08-api.yml | 2 +- kubernetes/09-dashboards-helper.yml | 2 +- kubernetes/10-zeek.yml | 4 +- kubernetes/11-suricata.yml | 4 +- kubernetes/12-file-monitor.yml | 4 +- kubernetes/13-filebeat.yml | 4 +- kubernetes/14-logstash.yml | 4 +- kubernetes/15-netbox-redis.yml | 4 +- kubernetes/16-netbox-redis-cache.yml | 2 +- kubernetes/17-netbox-postgres.yml | 4 +- kubernetes/18-netbox.yml | 4 +- kubernetes/19-htadmin.yml | 4 +- kubernetes/20-pcap-capture.yml | 4 +- kubernetes/21-zeek-live.yml | 4 +- kubernetes/22-suricata-live.yml | 4 +- kubernetes/23-arkime-live.yml | 4 +- kubernetes/24-freq.yml | 2 +- kubernetes/98-nginx-proxy.yml | 4 +- .../aws/ami/packer_vars.json.example | 2 +- 34 files changed, 177 insertions(+), 177 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 313cce697..f2fb41ff7 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -15,7 +15,7 @@ services: build: context: . dockerfile: Dockerfiles/opensearch.Dockerfile - image: ghcr.io/idaholab/malcolm/opensearch:24.02.1 + image: ghcr.io/idaholab/malcolm/opensearch:24.03.0 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -60,7 +60,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards-helper.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.02.1 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -93,7 +93,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards:24.02.1 + image: ghcr.io/idaholab/malcolm/dashboards:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -124,7 +124,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: ghcr.io/idaholab/malcolm/logstash-oss:24.02.1 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -178,7 +178,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.02.1 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -216,7 +216,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:24.02.1 + image: ghcr.io/idaholab/malcolm/arkime:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -255,7 +255,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:24.02.1 + image: ghcr.io/idaholab/malcolm/arkime:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -294,7 +294,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:24.02.1 + image: ghcr.io/idaholab/malcolm/zeek:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -330,7 +330,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:24.02.1 + image: ghcr.io/idaholab/malcolm/zeek:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -362,7 +362,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:24.02.1 + image: ghcr.io/idaholab/malcolm/suricata:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -393,7 +393,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:24.02.1 + image: ghcr.io/idaholab/malcolm/suricata:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -426,7 +426,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/file-monitor:24.02.1 + image: ghcr.io/idaholab/malcolm/file-monitor:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -458,7 +458,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-capture:24.02.1 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -490,7 +490,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.02.1 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -521,7 +521,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: ghcr.io/idaholab/malcolm/file-upload:24.02.1 + image: ghcr.io/idaholab/malcolm/file-upload:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -548,7 +548,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:24.02.1 + image: ghcr.io/idaholab/malcolm/htadmin:24.03.0 profiles: ["malcolm"] logging: *default-logging build: @@ -578,7 +578,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:24.02.1 + image: ghcr.io/idaholab/malcolm/freq:24.03.0 profiles: ["malcolm"] logging: *default-logging build: @@ -605,7 +605,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:24.02.1 + image: ghcr.io/idaholab/malcolm/netbox:24.03.0 profiles: ["malcolm"] logging: *default-logging build: @@ -642,7 +642,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:24.02.1 + image: ghcr.io/idaholab/malcolm/postgresql:24.03.0 profiles: ["malcolm"] logging: *default-logging build: @@ -671,7 +671,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:24.02.1 + image: ghcr.io/idaholab/malcolm/redis:24.03.0 profiles: ["malcolm"] logging: *default-logging build: @@ -704,7 +704,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:24.02.1 + image: ghcr.io/idaholab/malcolm/redis:24.03.0 profiles: ["malcolm"] logging: *default-logging build: @@ -736,7 +736,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:24.02.1 + image: ghcr.io/idaholab/malcolm/api:24.03.0 profiles: ["malcolm"] logging: *default-logging build: @@ -769,7 +769,7 @@ services: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.02.1 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" diff --git a/docker-compose.yml b/docker-compose.yml index 4cb47364f..51077bea9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ x-logging: services: opensearch: - image: ghcr.io/idaholab/malcolm/opensearch:24.02.1 + image: ghcr.io/idaholab/malcolm/opensearch:24.03.0 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -54,7 +54,7 @@ services: retries: 3 start_period: 180s dashboards-helper: - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.02.1 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -84,7 +84,7 @@ services: retries: 3 start_period: 30s dashboards: - image: ghcr.io/idaholab/malcolm/dashboards:24.02.1 + image: ghcr.io/idaholab/malcolm/dashboards:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -112,7 +112,7 @@ services: retries: 3 start_period: 210s logstash: - image: ghcr.io/idaholab/malcolm/logstash-oss:24.02.1 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -159,7 +159,7 @@ services: retries: 3 start_period: 600s filebeat: - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.02.1 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -194,7 +194,7 @@ services: retries: 3 start_period: 60s arkime: - image: ghcr.io/idaholab/malcolm/arkime:24.02.1 + image: ghcr.io/idaholab/malcolm/arkime:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -228,7 +228,7 @@ services: retries: 3 start_period: 210s arkime-live: - image: ghcr.io/idaholab/malcolm/arkime:24.02.1 + image: ghcr.io/idaholab/malcolm/arkime:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -261,7 +261,7 @@ services: - ./arkime/rules:/opt/arkime/rules:ro - ./pcap:/data/pcap zeek: - image: ghcr.io/idaholab/malcolm/zeek:24.02.1 + image: ghcr.io/idaholab/malcolm/zeek:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -293,7 +293,7 @@ services: retries: 3 start_period: 60s zeek-live: - image: ghcr.io/idaholab/malcolm/zeek:24.02.1 + image: ghcr.io/idaholab/malcolm/zeek:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -321,7 +321,7 @@ services: - ./zeek/intel:/opt/zeek/share/zeek/site/intel - ./zeek/custom:/opt/zeek/share/zeek/site/custom:ro suricata: - image: ghcr.io/idaholab/malcolm/suricata:24.02.1 + image: ghcr.io/idaholab/malcolm/suricata:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -349,7 +349,7 @@ services: retries: 3 start_period: 120s suricata-live: - image: ghcr.io/idaholab/malcolm/suricata:24.02.1 + image: ghcr.io/idaholab/malcolm/suricata:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -379,7 +379,7 @@ services: - ./suricata/rules:/opt/suricata/rules:ro - ./suricata/include-configs:/opt/suricata/include-configs:ro file-monitor: - image: ghcr.io/idaholab/malcolm/file-monitor:24.02.1 + image: ghcr.io/idaholab/malcolm/file-monitor:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -408,7 +408,7 @@ services: retries: 3 start_period: 60s pcap-capture: - image: ghcr.io/idaholab/malcolm/pcap-capture:24.02.1 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -437,7 +437,7 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./pcap/upload:/pcap pcap-monitor: - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.02.1 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -465,7 +465,7 @@ services: retries: 3 start_period: 90s upload: - image: ghcr.io/idaholab/malcolm/file-upload:24.02.1 + image: ghcr.io/idaholab/malcolm/file-upload:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -492,7 +492,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:24.02.1 + image: ghcr.io/idaholab/malcolm/htadmin:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -519,7 +519,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:24.02.1 + image: ghcr.io/idaholab/malcolm/freq:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -543,7 +543,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:24.02.1 + image: ghcr.io/idaholab/malcolm/netbox:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -577,7 +577,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:24.02.1 + image: ghcr.io/idaholab/malcolm/postgresql:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -603,7 +603,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:24.02.1 + image: ghcr.io/idaholab/malcolm/redis:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -633,7 +633,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:24.02.1 + image: ghcr.io/idaholab/malcolm/redis:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -662,7 +662,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:24.02.1 + image: ghcr.io/idaholab/malcolm/api:24.03.0 profiles: ["malcolm"] logging: *default-logging command: gunicorn --bind 0:5000 manage:app @@ -689,7 +689,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.02.1 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" diff --git a/docs/api-version.md b/docs/api-version.md index e281fba1a..7e4c08db5 100644 --- a/docs/api-version.md +++ b/docs/api-version.md @@ -47,6 +47,6 @@ Returns version information about Malcolm and version/[health](https://opensearc } }, "sha": "77574975", - "version": "24.02.1" + "version": "24.03.0" } ``` diff --git a/docs/contributing-pcap.md b/docs/contributing-pcap.md index df4961b76..adac63fb4 100644 --- a/docs/contributing-pcap.md +++ b/docs/contributing-pcap.md @@ -1,6 +1,6 @@ # PCAP processors -When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v24.02.1 release]({{ site.github.repository_url }}/releases/tag/v24.02.1)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: +When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v24.03.0 release]({{ site.github.repository_url }}/releases/tag/v24.03.0)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: 1. Define the service as instructed in the [Adding a new service](contributing-new-image.md#NewImage) section * Note how the existing `zeek` and `arkime` services use [bind mounts](contributing-local-modifications.md#Bind) to access the local `./pcap` directory diff --git a/docs/download.md b/docs/download.md index e7ad1cbe3..c52d6c540 100644 --- a/docs/download.md +++ b/docs/download.md @@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [malcolm-24.02.1.iso](/iso/malcolm-24.02.1.iso) (5.1GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/malcolm-24.02.1.iso.sha256.txt) | +| [malcolm-24.03.0.iso](/iso/malcolm-24.03.0.iso) (5.1GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/malcolm-24.03.0.iso.sha256.txt) | ## Hedgehog Linux @@ -26,7 +26,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [hedgehog-24.02.1.iso](/iso/hedgehog-24.02.1.iso) (2.5GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/hedgehog-24.02.1.iso.sha256.txt) | +| [hedgehog-24.03.0.iso](/iso/hedgehog-24.03.0.iso) (2.5GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/hedgehog-24.03.0.iso.sha256.txt) | ### Raspberry Pi 4 Image @@ -34,7 +34,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | Image | SHA256 | |---|---| -| [hedgehog-24.02.1_raspi_4.img.xz](/iso/hedgehog-24.02.1_raspi_4.img.xz) (1.4GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/hedgehog-24.02.1_raspi_4.img.xz.sha256.txt) | +| [hedgehog-24.03.0_raspi_4.img.xz](/iso/hedgehog-24.03.0_raspi_4.img.xz) (1.4GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/hedgehog-24.03.0_raspi_4.img.xz.sha256.txt) | ## Warning diff --git a/docs/hedgehog-iso-build.md b/docs/hedgehog-iso-build.md index e7a700f43..94a8877fd 100644 --- a/docs/hedgehog-iso-build.md +++ b/docs/hedgehog-iso-build.md @@ -29,7 +29,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-24.02.1.iso" +Finished, created "/sensor-build/hedgehog-24.03.0.iso" … ``` diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 8075932a7..c74fb5d94 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -272,28 +272,28 @@ agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | 861.34m | 14.36% | 19.55Gi | 9.29Gi | 61.28Gi | 11 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:24.02.1 | -file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:24.02.1 | -zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:24.02.1 | -dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:24.02.1 | -upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:24.02.1 | -filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:24.02.1 | -zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:24.02.1 | -logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:24.02.1 | -netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:24.02.1 | -suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:24.02.1 | -dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:24.02.1 | -netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:24.02.1 | -suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:24.02.1 | -freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:24.02.1 | -arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:24.02.1 | -pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:24.02.1 | -pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:24.02.1 | -netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:24.02.1 | -htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:24.02.1 | -netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:24.02.1 | -nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:24.02.1 | -opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:24.02.1 | +api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:24.03.0 | +file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:24.03.0 | +zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:24.03.0 | +dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:24.03.0 | +upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:24.03.0 | +filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:24.03.0 | +zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:24.03.0 | +logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:24.03.0 | +netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:24.03.0 | +suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:24.03.0 | +dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:24.03.0 | +netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:24.03.0 | +suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:24.03.0 | +freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:24.03.0 | +arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:24.03.0 | +pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:24.03.0 | +pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:24.03.0 | +netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:24.03.0 | +htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:24.03.0 | +netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:24.03.0 | +nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:24.03.0 | +opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:24.03.0 | ``` The other control scripts (`stop`, `restart`, `logs`, etc.) work in a similar manner as in a Docker-based deployment. One notable difference is the `wipe` script: data on PersistentVolume storage cannot be deleted by `wipe`. It must be deleted manually on the storage media underlying the PersistentVolumes. @@ -553,28 +553,28 @@ agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | 552.71m | 9.21% | 19.55Gi | 13.27Gi | 61.28Gi | 12 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:24.02.1 | -netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:24.02.1 | -dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:24.02.1 | -freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:24.02.1 | -pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:24.02.1 | -nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:24.02.1 | -htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:24.02.1 | -opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:24.02.1 | -zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:24.02.1 | -dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:24.02.1 | -arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:24.02.1 | -api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:24.02.1 | -netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:24.02.1 | -pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:24.02.1 | -suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:24.02.1 | -suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:24.02.1 | -netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:24.02.1 | -zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:24.02.1 | -filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:24.02.1 | -file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:24.02.1 | -upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:24.02.1 | -logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:24.02.1 | +netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:24.03.0 | +netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:24.03.0 | +dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:24.03.0 | +freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:24.03.0 | +pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:24.03.0 | +nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:24.03.0 | +htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:24.03.0 | +opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:24.03.0 | +zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:24.03.0 | +dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:24.03.0 | +arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:24.03.0 | +api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:24.03.0 | +netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:24.03.0 | +pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:24.03.0 | +suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:24.03.0 | +suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:24.03.0 | +netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:24.03.0 | +zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:24.03.0 | +filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:24.03.0 | +file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:24.03.0 | +upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:24.03.0 | +logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:24.03.0 | ``` View container logs for the Malcolm deployment with `./scripts/logs` (if **[stern](https://github.com/stern/stern)** present in `$PATH`): diff --git a/docs/malcolm-iso.md b/docs/malcolm-iso.md index d2e3e627e..21ec5f9eb 100644 --- a/docs/malcolm-iso.md +++ b/docs/malcolm-iso.md @@ -41,7 +41,7 @@ Building the ISO may take 30 minutes or more depending on the system. As the bui ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-24.02.1.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-24.03.0.iso" … ``` diff --git a/docs/quickstart.md b/docs/quickstart.md index dc50aa7d9..8d4e5a8a9 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -54,25 +54,25 @@ You can then observe the images have been retrieved by running `docker images`: ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 24.02.1 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 24.02.1 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 24.02.1 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 24.02.1 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 24.02.1 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 24.02.1 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.02.1 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 24.02.1 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 24.02.1 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 24.02.1 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 24.02.1 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 24.02.1 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 24.02.1 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 24.02.1 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 24.02.1 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 24.02.1 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 24.02.1 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 24.02.1 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 24.02.1 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 24.03.0 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 24.03.0 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 24.03.0 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 24.03.0 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 24.03.0 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 24.03.0 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.03.0 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 24.03.0 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 24.03.0 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 24.03.0 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 24.03.0 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 24.03.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 24.03.0 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 24.03.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.03.0 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 24.03.0 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 24.03.0 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 24.03.0 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 24.03.0 xxxxxxxxxxxx 3 days ago 1GB ``` ### Import from pre-packaged tarballs diff --git a/docs/ubuntu-install-example.md b/docs/ubuntu-install-example.md index 69705bd29..ff4f0d4b0 100644 --- a/docs/ubuntu-install-example.md +++ b/docs/ubuntu-install-example.md @@ -257,25 +257,25 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 24.02.1 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 24.02.1 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 24.02.1 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 24.02.1 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 24.02.1 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 24.02.1 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.02.1 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 24.02.1 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 24.02.1 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 24.02.1 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 24.02.1 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 24.02.1 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 24.02.1 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 24.02.1 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 24.02.1 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 24.02.1 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 24.02.1 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 24.02.1 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 24.02.1 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 24.03.0 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 24.03.0 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 24.03.0 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 24.03.0 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 24.03.0 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 24.03.0 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.03.0 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 24.03.0 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 24.03.0 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 24.03.0 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 24.03.0 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 24.03.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 24.03.0 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 24.03.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.03.0 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 24.03.0 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 24.03.0 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 24.03.0 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 24.03.0 xxxxxxxxxxxx 3 days ago 1GB ``` Finally, start Malcolm. When Malcolm starts it will stream informational and debug messages to the console until it has completed initializing. diff --git a/kubernetes/03-opensearch.yml b/kubernetes/03-opensearch.yml index 53f9baa5b..ec9b8dd56 100644 --- a/kubernetes/03-opensearch.yml +++ b/kubernetes/03-opensearch.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: opensearch-container - image: ghcr.io/idaholab/malcolm/opensearch:24.02.1 + image: ghcr.io/idaholab/malcolm/opensearch:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -71,7 +71,7 @@ spec: subPath: "opensearch" initContainers: - name: opensearch-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/04-dashboards.yml b/kubernetes/04-dashboards.yml index 82b4d0e42..f70cb83aa 100644 --- a/kubernetes/04-dashboards.yml +++ b/kubernetes/04-dashboards.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-container - image: ghcr.io/idaholab/malcolm/dashboards:24.02.1 + image: ghcr.io/idaholab/malcolm/dashboards:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/05-upload.yml b/kubernetes/05-upload.yml index 70616e5f0..e0aeb3745 100644 --- a/kubernetes/05-upload.yml +++ b/kubernetes/05-upload.yml @@ -34,7 +34,7 @@ spec: spec: containers: - name: upload-container - image: ghcr.io/idaholab/malcolm/file-upload:24.02.1 + image: ghcr.io/idaholab/malcolm/file-upload:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -73,7 +73,7 @@ spec: subPath: "upload" initContainers: - name: upload-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/06-pcap-monitor.yml b/kubernetes/06-pcap-monitor.yml index 675597120..427bb4d7c 100644 --- a/kubernetes/06-pcap-monitor.yml +++ b/kubernetes/06-pcap-monitor.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: pcap-monitor-container - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.02.1 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -70,7 +70,7 @@ spec: name: pcap-monitor-zeek-volume initContainers: - name: pcap-monitor-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/07-arkime.yml b/kubernetes/07-arkime.yml index c2984cc06..9085b877d 100644 --- a/kubernetes/07-arkime.yml +++ b/kubernetes/07-arkime.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: arkime-container - image: ghcr.io/idaholab/malcolm/arkime:24.02.1 + image: ghcr.io/idaholab/malcolm/arkime:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -79,7 +79,7 @@ spec: name: arkime-pcap-volume initContainers: - name: arkime-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/08-api.yml b/kubernetes/08-api.yml index 36f5d7a39..584799bd5 100644 --- a/kubernetes/08-api.yml +++ b/kubernetes/08-api.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: api-container - image: ghcr.io/idaholab/malcolm/api:24.02.1 + image: ghcr.io/idaholab/malcolm/api:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/09-dashboards-helper.yml b/kubernetes/09-dashboards-helper.yml index b3223f374..3ea236ed1 100644 --- a/kubernetes/09-dashboards-helper.yml +++ b/kubernetes/09-dashboards-helper.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-helper-container - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.02.1 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index 02e84f99b..10827bb57 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-offline-container - image: ghcr.io/idaholab/malcolm/zeek:24.02.1 + image: ghcr.io/idaholab/malcolm/zeek:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -64,7 +64,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/11-suricata.yml b/kubernetes/11-suricata.yml index f838bd17b..80de0fed8 100644 --- a/kubernetes/11-suricata.yml +++ b/kubernetes/11-suricata.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-offline-container - image: ghcr.io/idaholab/malcolm/suricata:24.02.1 + image: ghcr.io/idaholab/malcolm/suricata:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -55,7 +55,7 @@ spec: name: suricata-offline-custom-configs-volume initContainers: - name: suricata-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/12-file-monitor.yml b/kubernetes/12-file-monitor.yml index 1bd47d08d..647cddedd 100644 --- a/kubernetes/12-file-monitor.yml +++ b/kubernetes/12-file-monitor.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: file-monitor-container - image: ghcr.io/idaholab/malcolm/file-monitor:24.02.1 + image: ghcr.io/idaholab/malcolm/file-monitor:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: name: file-monitor-yara-rules-custom-volume initContainers: - name: file-monitor-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/13-filebeat.yml b/kubernetes/13-filebeat.yml index c14bbe34e..edb237b3f 100644 --- a/kubernetes/13-filebeat.yml +++ b/kubernetes/13-filebeat.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: filebeat-container - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.02.1 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: "nginx" initContainers: - name: filebeat-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/14-logstash.yml b/kubernetes/14-logstash.yml index 7239101bf..cfe42b9b2 100644 --- a/kubernetes/14-logstash.yml +++ b/kubernetes/14-logstash.yml @@ -49,7 +49,7 @@ spec: # topologyKey: "kubernetes.io/hostname" containers: - name: logstash-container - image: ghcr.io/idaholab/malcolm/logstash-oss:24.02.1 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -115,7 +115,7 @@ spec: subPath: "logstash" initContainers: - name: logstash-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/15-netbox-redis.yml b/kubernetes/15-netbox-redis.yml index a85550502..206a486a9 100644 --- a/kubernetes/15-netbox-redis.yml +++ b/kubernetes/15-netbox-redis.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-container - image: ghcr.io/idaholab/malcolm/redis:24.02.1 + image: ghcr.io/idaholab/malcolm/redis:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: netbox/redis initContainers: - name: netbox-redis-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/16-netbox-redis-cache.yml b/kubernetes/16-netbox-redis-cache.yml index e1aaa41b6..a7985dfb1 100644 --- a/kubernetes/16-netbox-redis-cache.yml +++ b/kubernetes/16-netbox-redis-cache.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-cache-container - image: ghcr.io/idaholab/malcolm/redis:24.02.1 + image: ghcr.io/idaholab/malcolm/redis:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/17-netbox-postgres.yml b/kubernetes/17-netbox-postgres.yml index 3f40e3cd6..a9e2cab83 100644 --- a/kubernetes/17-netbox-postgres.yml +++ b/kubernetes/17-netbox-postgres.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-postgres-container - image: ghcr.io/idaholab/malcolm/postgresql:24.02.1 + image: ghcr.io/idaholab/malcolm/postgresql:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -74,7 +74,7 @@ spec: subPath: netbox/postgres initContainers: - name: netbox-postgres-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index 75475df24..f2a3469c5 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -36,7 +36,7 @@ spec: spec: containers: - name: netbox-container - image: ghcr.io/idaholab/malcolm/netbox:24.02.1 + image: ghcr.io/idaholab/malcolm/netbox:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -88,7 +88,7 @@ spec: subPath: netbox/media initContainers: - name: netbox-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/19-htadmin.yml b/kubernetes/19-htadmin.yml index c2bfaaea1..9bb3f0736 100644 --- a/kubernetes/19-htadmin.yml +++ b/kubernetes/19-htadmin.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: htadmin-container - image: ghcr.io/idaholab/malcolm/htadmin:24.02.1 + image: ghcr.io/idaholab/malcolm/htadmin:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -63,7 +63,7 @@ spec: subPath: "htadmin" initContainers: - name: htadmin-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index e430dd8fd..7656623d9 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: pcap-capture-container - image: ghcr.io/idaholab/malcolm/pcap-capture:24.02.1 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -50,7 +50,7 @@ spec: subPath: "upload" initContainers: - name: pcap-capture-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index 67e7e736b..b4e220e20 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-live-container - image: ghcr.io/idaholab/malcolm/zeek:24.02.1 + image: ghcr.io/idaholab/malcolm/zeek:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -61,7 +61,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index 17e2204be..3490b1dbe 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-live-container - image: ghcr.io/idaholab/malcolm/suricata:24.02.1 + image: ghcr.io/idaholab/malcolm/suricata:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -56,7 +56,7 @@ spec: name: suricata-live-custom-configs-volume initContainers: - name: suricata-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/23-arkime-live.yml b/kubernetes/23-arkime-live.yml index 0429c6ea8..0ce28c8a9 100644 --- a/kubernetes/23-arkime-live.yml +++ b/kubernetes/23-arkime-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: arkime-live-container - image: ghcr.io/idaholab/malcolm/arkime:24.02.1 + image: ghcr.io/idaholab/malcolm/arkime:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -62,7 +62,7 @@ spec: name: arkime-live-pcap-volume initContainers: - name: arkime-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/24-freq.yml b/kubernetes/24-freq.yml index 9f158303e..930bfacb7 100644 --- a/kubernetes/24-freq.yml +++ b/kubernetes/24-freq.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: freq-container - image: ghcr.io/idaholab/malcolm/freq:24.02.1 + image: ghcr.io/idaholab/malcolm/freq:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/98-nginx-proxy.yml b/kubernetes/98-nginx-proxy.yml index 61847052e..7b3ec0ae0 100644 --- a/kubernetes/98-nginx-proxy.yml +++ b/kubernetes/98-nginx-proxy.yml @@ -39,7 +39,7 @@ spec: spec: containers: - name: nginx-proxy-container - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.02.1 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -99,7 +99,7 @@ spec: subPath: "nginx" initContainers: - name: nginx-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/scripts/third-party-environments/aws/ami/packer_vars.json.example b/scripts/third-party-environments/aws/ami/packer_vars.json.example index 59cc8544b..6243c5796 100644 --- a/scripts/third-party-environments/aws/ami/packer_vars.json.example +++ b/scripts/third-party-environments/aws/ami/packer_vars.json.example @@ -2,7 +2,7 @@ "aws_access_key": "XXXXXXXXXXXXXXXXXXXX", "aws_secret_key": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "instance_type": "t2.micro", - "malcolm_tag": "v24.02.1", + "malcolm_tag": "v24.03.0", "malcolm_repo": "idaholab/Malcolm", "malcolm_uid": "1000", "ssh_username": "ec2-user", From 2c6c86e5317592816bba45c4cc17197984146a15 Mon Sep 17 00:00:00 2001 From: SG Date: Fri, 16 Feb 2024 11:38:03 -0700 Subject: [PATCH 03/79] renamed sensor-iso to hedgehog-iso and sensor-raspi to hedgehog-raspi --- .dockerignore | 4 +-- ...gehog-iso-build-docker-wrap-push-ghcr.yml} | 24 +++++++++--------- ...hog-raspi-build-docker-wrap-push-ghcr.yml} | 6 ++--- .gitignore | 14 +++++----- _config.yml | 2 +- docs/development.md | 3 ++- docs/hedgehog-iso-build.md | 4 +-- docs/hedgehog-raspi-build.md | 4 +-- docs/hedgehog-upgrade.md | 6 ++--- docs/hedgehog.md | 2 +- docs/malcolm-hedgehog-e2e-iso-install.md | 2 +- {sensor-iso => hedgehog-iso}/.dockerignore | 0 {sensor-iso => hedgehog-iso}/.gitignore | 0 {sensor-iso => hedgehog-iso}/Dockerfile | 2 +- .../arkime/Dockerfile | 0 .../arkime/build-arkime-deb.sh | 0 .../arkime/build-docker-image.sh | 0 .../arkime/etc/.gitignore | 0 {sensor-iso => hedgehog-iso}/build.sh | 0 .../build_via_vagrant.sh | 0 .../config/archives/beats.key.binary | 0 .../config/archives/beats.key.chroot | 0 .../config/archives/beats.list.binary | 0 .../config/archives/beats.list.chroot | 0 .../config/archives/fluentbit.key.binary | 0 .../config/archives/fluentbit.key.chroot | 0 .../config/archives/fluentbit.list.binary | 0 .../config/archives/fluentbit.list.chroot | 0 .../config/archives/mozilla.key.binary | 0 .../config/archives/mozilla.key.chroot | 0 .../config/archives/mozilla.list.binary | 0 .../config/archives/mozilla.list.chroot | 0 .../config/archives/mozilla.pref.binary | 0 .../config/archives/mozilla.pref.chroot | 0 .../config/bootloaders/extlinux/extlinux.conf | 0 .../config/bootloaders/grub-legacy/menu.lst | 0 .../bootloaders/grub-legacy/splash.xpm.gz | Bin .../config/bootloaders/grub-pc/config.cfg | 0 .../bootloaders/grub-pc/dejavu-bold-14.pf2 | Bin .../bootloaders/grub-pc/dejavu-bold-16.pf2 | Bin .../config/bootloaders/grub-pc/grub.cfg | 0 .../bootloaders/grub-pc/hedgehog-color.png | Bin .../grub-pc/install_start_text.cfg | 0 .../bootloaders/grub-pc/install_text.cfg | 0 .../bootloaders/grub-pc/live-theme/theme.txt | 0 .../config/bootloaders/grub-pc/memtest.cfg | 0 .../config/bootloaders/grub-pc/splash.png | Bin .../config/bootloaders/grub-pc/theme.cfg | 0 .../config/bootloaders/isolinux/isolinux.cfg | 0 .../bootloaders/pxelinux/pxelinux.cfg/default | 0 .../config/bootloaders/syslinux/syslinux.cfg | 0 .../syslinux_common/install_text.cfg | 0 .../bootloaders/syslinux_common/live.cfg.in | 0 .../bootloaders/syslinux_common/memtest.cfg | 0 .../bootloaders/syslinux_common/menu.cfg | 0 .../bootloaders/syslinux_common/splash.svg | 0 .../bootloaders/syslinux_common/stdmenu.cfg | 0 .../bootloaders/syslinux_common/utilities.cfg | 0 .../normal/0169-pip-installs.hook.chroot | 0 .../normal/0900-setup-rc-local.hook.chroot | 0 .../normal/0910-sensor-build.hook.chroot | 0 .../normal/0911-get-stig-scripts.hook.chroot | 0 .../0990-remove-unwanted-pkg.hook.chroot | 0 .../0991-security-performance.hook.chroot | 0 .../hooks/normal/0992-localepurge.hook.chroot | 0 .../hooks/normal/0992-login.hook.chroot | 0 .../includes.binary/install/preseed_base.cfg | 0 .../install/preseed_multipar.cfg | 0 .../install/preseed_vmware.cfg | 0 .../live/filesystem.packages-remove | 0 .../etc/audit/rules.d/audit.rules | 0 .../includes.chroot/etc/bash.bash_aliases | 0 .../includes.chroot/etc/bash.bash_functions | 0 .../config/includes.chroot/etc/installer | 0 .../etc/lightdm/slick-greeter.conf | 0 .../includes.chroot/etc/live/config.conf | 0 .../config/includes.chroot/etc/login.defs | 0 .../etc/security/limits.d/limits.conf | 0 .../config/includes.chroot/etc/skel/.bashrc | 0 .../autostart/capture-format-wait.desktop | 0 .../autostart/dod-login-banner.desktop | 0 .../.config/autostart/hedgehog-kiosk.desktop | 0 .../set-dconf-screen-lock-defaults.desktop | 0 .../etc/skel/.config/dconf/user | Bin .../etc/skel/.config/gtk-3.0/bookmarks | 0 .../etc/skel/.config/gtk-3.0/settings.ini | 0 .../.config/light-locker-dconf-defaults.conf | 0 .../etc/skel/.config/mimeapps.list | 0 .../etc/skel/.config/xfce4/helpers.rc | 0 .../panel/launcher-15/16346759461.desktop | 0 .../panel/launcher-16/16346759562.desktop | 0 .../panel/launcher-17/16346759653.desktop | 0 .../panel/launcher-18/16346759724.desktop | 0 .../panel/launcher-19/16346759855.desktop | 0 .../panel/launcher-20/16346759956.desktop | 0 .../panel/launcher-21/16346760027.desktop | 0 .../panel/launcher-22/16346760098.desktop | 0 .../skel/.config/xfce4/terminal/terminalrc | 0 .../xfce-perchannel-xml/xfce4-desktop.xml | 0 .../xfce-perchannel-xml/xfce4-panel.xml | 0 .../xfce4-power-manager.xml | 0 .../xfce-perchannel-xml/xfce4-session.xml | 0 .../xfconf/xfce-perchannel-xml/xfwm4.xml | 0 .../xfconf/xfce-perchannel-xml/xsettings.xml | 0 .../includes.chroot/etc/skel/.hushlogin | 0 .../helpers/custom-TerminalEmulator.desktop | 0 .../includes.chroot/etc/skel/.selected_editor | 0 .../includes.chroot/etc/skel/.tmux.conf | 0 .../config/includes.chroot/etc/skel/.vimrc | 0 .../includes.chroot/etc/ssh/sshd_config | 0 .../etc/sudoers.d/aide_integrity_check | 0 .../etc/sudoers.d/nic_capture_setup | 0 .../etc/sudoers.d/ufw_arkime_viewer | 0 .../systemd/system/sensor-shutdown.service | 0 .../opt/zeek/share/zeek/site/extractor.zeek | 0 .../share/zeek/site/extractor_params.zeek | 0 .../includes.chroot/root/.selected_editor | 0 .../config/includes.chroot/root/.vimrc | 0 .../usr/local/bin/hedgehog-kiosk.sh | 0 .../usr/local/bin/sensor-shutdown | 0 .../usr/local/etc/zeek/guess.zeek | 0 .../usr/local/etc/zeek/guess_ics_map.txt | 0 .../usr/local/etc/zeek/local.zeek | 0 .../usr/local/etc/zeek/login.zeek | 0 .../applications/configure-capture.desktop | 0 .../applications/configure-interfaces.desktop | 0 .../share/applications/hedgehog-kiosk.desktop | 0 .../applications/hedgehog-readme.desktop | 0 .../sensor-services-full-restart.desktop | 0 .../sensor-services-status.desktop | 0 .../config/package-lists/build.list.chroot | 0 .../package-lists/desktopmanager.list.chroot | 0 .../config/package-lists/grub.list.binary | 0 .../config/package-lists/live.list.chroot | 0 .../config/package-lists/net.list.chroot | 0 .../config/package-lists/python.list.chroot | 0 .../config/package-lists/sensor.list.chroot | 0 .../config/package-lists/system.list.chroot | 0 .../package-lists/virtualguest.list.chroot | 0 .../interface/.gitignore | 0 .../interface/init.sh | 0 .../interface/kiosk.service | 0 .../interface/requirements.txt | 0 .../interface/sensor_ctl/arkime/config.ini | 0 .../interface/sensor_ctl/arkime/rules.yml | 0 .../interface/sensor_ctl/clean | 0 .../interface/sensor_ctl/clean.sh | 0 .../interface/sensor_ctl/control.sh | 0 .../interface/sensor_ctl/control_vars.conf | 0 .../sensor_ctl/filebeat/filebeat.yml | 0 .../filebeat/sensor_filebeat_local.sh | 0 .../logstash-client-certificates/.gitignore | 0 .../sensor_ctl/miscbeat/filebeat.yml | 0 .../miscbeat/sensor_miscbeat_local.sh | 0 .../interface/sensor_ctl/reload | 0 .../interface/sensor_ctl/restart | 0 .../sensor_ctl/scripts/log_disk_space.sh | 0 .../interface/sensor_ctl/shutdown | 0 .../interface/sensor_ctl/start | 0 .../interface/sensor_ctl/status | 0 .../interface/sensor_ctl/stop | 0 .../sensor_ctl/supervisor.d/arkime.conf | 0 .../sensor_ctl/supervisor.d/beats.conf | 0 .../sensor_ctl/supervisor.d/clamav.conf | 0 .../supervisor.d/documentation.conf | 0 .../supervisor.d/fluentbit-aide.conf | 0 .../supervisor.d/fluentbit-auditlog.conf | 0 .../supervisor.d/fluentbit-kmsg.conf | 0 .../supervisor.d/fluentbit-metrics.conf | 0 .../supervisor.d/fluentbit-systemd.conf | 0 .../supervisor.d/fluentbit-thermal.conf | 0 .../sensor_ctl/supervisor.d/netsniff.template | 0 .../sensor_ctl/supervisor.d/prune.conf | 0 .../sensor_ctl/supervisor.d/supercronic.conf | 0 .../sensor_ctl/supervisor.d/suricata.conf | 0 .../sensor_ctl/supervisor.d/tcpdump.template | 0 .../sensor_ctl/supervisor.d/zeek.conf | 0 .../supervisor.init/arkime_config_populate.sh | 0 .../supervisor.init/supercronic_populate.sh | 0 .../suricata_config_populate.sh | 0 .../interface/sensor_ctl/supervisor.sh | 0 .../interface/sensor_ctl/supervisord.conf | 0 .../zeek/extractor_override.interesting.zeek | 0 .../sensor_ctl/zeek/intel/.gitignore | 0 .../interface/sensor_interface/__init__.py | 0 .../interface/sensor_interface/__main__.py | 0 .../interface/sensor_interface/routes.py | 0 .../sensor_interface/static/css/custom.css | 0 .../static/css/freeboard.min.css | 0 .../sensor_interface/static/css/material.css | 0 .../static/dashboard/dashboard.json | 0 .../sensor_interface/static/icons/icons.woff2 | Bin .../static/img/dropdown-arrow.png | Bin .../static/img/glyphicons-blackboard.png | Bin .../static/img/glyphicons-halflings-white.png | Bin .../static/img/glyphicons-halflings.png | Bin .../static/img/glyphicons-log-in.png | Bin .../static/img/glyphicons-log-out.png | Bin .../sensor_interface/static/js/custom.js | 0 .../static/js/freeboard.min.js | 0 .../static/js/freeboard.plugins.min.js | 0 .../static/js/freeboard.thirdparty.min.js | 0 .../static/js/freeboard_plugins.min.js | 0 .../static/js/jquery.sparkline.min.js | 0 .../static/js/material.min.js | 0 .../sensor_interface/sysquery/__init__.py | 0 .../sensor_interface/sysquery/sys_service.py | 0 .../sensor_interface/templates/base.html | 0 .../sensor_interface/templates/buttons.html | 0 .../templates/system_block.html | 0 .../vagrant/Vagrantfile | 0 .../vagrant/ssh_config | 0 {sensor-iso => hedgehog-iso}/yara/Dockerfile | 0 .../yara/build-docker-image.sh | 0 .../yara/build-yara-deb.sh | 0 {sensor-raspi => hedgehog-raspi}/.gitignore | 0 {sensor-raspi => hedgehog-raspi}/Dockerfile | 2 +- {sensor-raspi => hedgehog-raspi}/Makefile | 0 {sensor-raspi => hedgehog-raspi}/README.md | 2 +- .../attribution.txt | 0 .../build_via_vagrant.sh | 2 +- .../debian/salsa-ci.yml | 0 .../debian/yamllint.yml | 0 .../generate-recipe.py | 2 +- .../raspi_master.yaml | 0 .../rootfs/boot/firmware/sysconf.txt | 0 .../rootfs/etc/fstab | 0 .../initramfs-tools/hooks/rpi-resizerootfs | 0 .../scripts/local-bottom/rpi-resizerootfs | 0 .../rootfs/etc/network/interfaces.d/eth0 | 0 .../rootfs/etc/network/interfaces.d/wlan0 | 0 .../system/rpi-generate-ssh-host-keys.service | 0 .../rpi-reconfigure-raspi-firmware.service | 0 .../systemd/system/rpi-set-sysconf.service | 0 .../etc/wpa_supplicant/wpa_supplicant.conf | 0 .../rootfs/usr/local/sbin/rpi-set-sysconf | 0 .../sensor_install.sh | 2 +- .../vagrant/Vagrantfile | 0 .../vagrant/ssh_config | 0 239 files changed, 42 insertions(+), 41 deletions(-) rename .github/workflows/{sensor-iso-build-docker-wrap-push-ghcr.yml => hedgehog-iso-build-docker-wrap-push-ghcr.yml} (87%) rename .github/workflows/{sensor-raspi-build-docker-wrap-push-ghcr.yml => hedgehog-raspi-build-docker-wrap-push-ghcr.yml} (96%) rename {sensor-iso => hedgehog-iso}/.dockerignore (100%) rename {sensor-iso => hedgehog-iso}/.gitignore (100%) rename {sensor-iso => hedgehog-iso}/Dockerfile (95%) rename {sensor-iso => hedgehog-iso}/arkime/Dockerfile (100%) rename {sensor-iso => hedgehog-iso}/arkime/build-arkime-deb.sh (100%) rename {sensor-iso => hedgehog-iso}/arkime/build-docker-image.sh (100%) rename {sensor-iso => hedgehog-iso}/arkime/etc/.gitignore (100%) rename {sensor-iso => hedgehog-iso}/build.sh (100%) rename {sensor-iso => hedgehog-iso}/build_via_vagrant.sh (100%) rename {sensor-iso => hedgehog-iso}/config/archives/beats.key.binary (100%) rename {sensor-iso => hedgehog-iso}/config/archives/beats.key.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/archives/beats.list.binary (100%) rename {sensor-iso => hedgehog-iso}/config/archives/beats.list.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/archives/fluentbit.key.binary (100%) rename {sensor-iso => hedgehog-iso}/config/archives/fluentbit.key.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/archives/fluentbit.list.binary (100%) rename {sensor-iso => hedgehog-iso}/config/archives/fluentbit.list.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/archives/mozilla.key.binary (100%) rename {sensor-iso => hedgehog-iso}/config/archives/mozilla.key.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/archives/mozilla.list.binary (100%) rename {sensor-iso => hedgehog-iso}/config/archives/mozilla.list.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/archives/mozilla.pref.binary (100%) rename {sensor-iso => hedgehog-iso}/config/archives/mozilla.pref.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/extlinux/extlinux.conf (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/grub-legacy/menu.lst (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/grub-legacy/splash.xpm.gz (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/grub-pc/config.cfg (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/grub-pc/dejavu-bold-14.pf2 (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/grub-pc/dejavu-bold-16.pf2 (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/grub-pc/grub.cfg (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/grub-pc/hedgehog-color.png (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/grub-pc/install_start_text.cfg (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/grub-pc/install_text.cfg (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/grub-pc/live-theme/theme.txt (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/grub-pc/memtest.cfg (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/grub-pc/splash.png (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/grub-pc/theme.cfg (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/isolinux/isolinux.cfg (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/pxelinux/pxelinux.cfg/default (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/syslinux/syslinux.cfg (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/syslinux_common/install_text.cfg (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/syslinux_common/live.cfg.in (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/syslinux_common/memtest.cfg (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/syslinux_common/menu.cfg (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/syslinux_common/splash.svg (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/syslinux_common/stdmenu.cfg (100%) rename {sensor-iso => hedgehog-iso}/config/bootloaders/syslinux_common/utilities.cfg (100%) rename {sensor-iso => hedgehog-iso}/config/hooks/normal/0169-pip-installs.hook.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/hooks/normal/0900-setup-rc-local.hook.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/hooks/normal/0910-sensor-build.hook.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/hooks/normal/0911-get-stig-scripts.hook.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/hooks/normal/0991-security-performance.hook.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/hooks/normal/0992-localepurge.hook.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/hooks/normal/0992-login.hook.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/includes.binary/install/preseed_base.cfg (100%) rename {sensor-iso => hedgehog-iso}/config/includes.binary/install/preseed_multipar.cfg (100%) rename {sensor-iso => hedgehog-iso}/config/includes.binary/install/preseed_vmware.cfg (100%) rename {sensor-iso => hedgehog-iso}/config/includes.binary/live/filesystem.packages-remove (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/audit/rules.d/audit.rules (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/bash.bash_aliases (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/bash.bash_functions (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/installer (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/lightdm/slick-greeter.conf (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/live/config.conf (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/login.defs (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/security/limits.d/limits.conf (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.bashrc (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/autostart/capture-format-wait.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/autostart/dod-login-banner.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/autostart/hedgehog-kiosk.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/autostart/set-dconf-screen-lock-defaults.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/dconf/user (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/gtk-3.0/bookmarks (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/gtk-3.0/settings.ini (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/light-locker-dconf-defaults.conf (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/mimeapps.list (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/xfce4/helpers.rc (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-15/16346759461.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-16/16346759562.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-17/16346759653.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-18/16346759724.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-19/16346759855.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-20/16346759956.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-21/16346760027.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-22/16346760098.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/xfce4/terminal/terminalrc (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-desktop.xml (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-power-manager.xml (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-session.xml (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfwm4.xml (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xsettings.xml (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.hushlogin (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.local/share/xfce4/helpers/custom-TerminalEmulator.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.selected_editor (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.tmux.conf (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/skel/.vimrc (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/ssh/sshd_config (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/sudoers.d/aide_integrity_check (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/sudoers.d/nic_capture_setup (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/etc/systemd/system/sensor-shutdown.service (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/opt/zeek/share/zeek/site/extractor.zeek (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/opt/zeek/share/zeek/site/extractor_params.zeek (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/root/.selected_editor (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/root/.vimrc (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/usr/local/bin/hedgehog-kiosk.sh (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/usr/local/bin/sensor-shutdown (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/usr/local/etc/zeek/guess.zeek (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/usr/local/etc/zeek/guess_ics_map.txt (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/usr/local/etc/zeek/local.zeek (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/usr/local/etc/zeek/login.zeek (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/usr/share/applications/configure-capture.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/usr/share/applications/configure-interfaces.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/usr/share/applications/hedgehog-kiosk.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/usr/share/applications/hedgehog-readme.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/usr/share/applications/sensor-services-full-restart.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/includes.chroot/usr/share/applications/sensor-services-status.desktop (100%) rename {sensor-iso => hedgehog-iso}/config/package-lists/build.list.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/package-lists/desktopmanager.list.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/package-lists/grub.list.binary (100%) rename {sensor-iso => hedgehog-iso}/config/package-lists/live.list.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/package-lists/net.list.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/package-lists/python.list.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/package-lists/sensor.list.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/package-lists/system.list.chroot (100%) rename {sensor-iso => hedgehog-iso}/config/package-lists/virtualguest.list.chroot (100%) rename {sensor-iso => hedgehog-iso}/interface/.gitignore (100%) rename {sensor-iso => hedgehog-iso}/interface/init.sh (100%) rename {sensor-iso => hedgehog-iso}/interface/kiosk.service (100%) rename {sensor-iso => hedgehog-iso}/interface/requirements.txt (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/arkime/config.ini (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/arkime/rules.yml (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/clean (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/clean.sh (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/control.sh (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/control_vars.conf (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/filebeat/filebeat.yml (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/filebeat/sensor_filebeat_local.sh (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/logstash-client-certificates/.gitignore (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/miscbeat/filebeat.yml (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/reload (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/restart (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/scripts/log_disk_space.sh (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/shutdown (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/start (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/status (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/stop (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.d/arkime.conf (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.d/beats.conf (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.d/clamav.conf (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.d/documentation.conf (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.d/fluentbit-aide.conf (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.d/fluentbit-auditlog.conf (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.d/fluentbit-kmsg.conf (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.d/fluentbit-metrics.conf (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.d/fluentbit-systemd.conf (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.d/fluentbit-thermal.conf (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.d/netsniff.template (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.d/prune.conf (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.d/supercronic.conf (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.d/suricata.conf (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.d/tcpdump.template (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.d/zeek.conf (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.init/supercronic_populate.sh (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisor.sh (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/supervisord.conf (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/zeek/extractor_override.interesting.zeek (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_ctl/zeek/intel/.gitignore (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/__init__.py (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/__main__.py (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/routes.py (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/static/css/custom.css (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/static/css/freeboard.min.css (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/static/css/material.css (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/static/dashboard/dashboard.json (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/static/icons/icons.woff2 (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/static/img/dropdown-arrow.png (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/static/img/glyphicons-blackboard.png (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/static/img/glyphicons-halflings-white.png (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/static/img/glyphicons-halflings.png (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/static/img/glyphicons-log-in.png (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/static/img/glyphicons-log-out.png (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/static/js/custom.js (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/static/js/freeboard.min.js (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/static/js/freeboard.plugins.min.js (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/static/js/freeboard.thirdparty.min.js (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/static/js/freeboard_plugins.min.js (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/static/js/jquery.sparkline.min.js (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/static/js/material.min.js (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/sysquery/__init__.py (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/sysquery/sys_service.py (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/templates/base.html (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/templates/buttons.html (100%) rename {sensor-iso => hedgehog-iso}/interface/sensor_interface/templates/system_block.html (100%) rename {sensor-iso => hedgehog-iso}/vagrant/Vagrantfile (100%) rename {sensor-iso => hedgehog-iso}/vagrant/ssh_config (100%) rename {sensor-iso => hedgehog-iso}/yara/Dockerfile (100%) rename {sensor-iso => hedgehog-iso}/yara/build-docker-image.sh (100%) rename {sensor-iso => hedgehog-iso}/yara/build-yara-deb.sh (100%) rename {sensor-raspi => hedgehog-raspi}/.gitignore (100%) rename {sensor-raspi => hedgehog-raspi}/Dockerfile (95%) rename {sensor-raspi => hedgehog-raspi}/Makefile (100%) rename {sensor-raspi => hedgehog-raspi}/README.md (99%) rename {sensor-raspi => hedgehog-raspi}/attribution.txt (100%) rename {sensor-raspi => hedgehog-raspi}/build_via_vagrant.sh (96%) rename {sensor-raspi => hedgehog-raspi}/debian/salsa-ci.yml (100%) rename {sensor-raspi => hedgehog-raspi}/debian/yamllint.yml (100%) rename {sensor-raspi => hedgehog-raspi}/generate-recipe.py (99%) rename {sensor-raspi => hedgehog-raspi}/raspi_master.yaml (100%) rename {sensor-raspi => hedgehog-raspi}/rootfs/boot/firmware/sysconf.txt (100%) rename {sensor-raspi => hedgehog-raspi}/rootfs/etc/fstab (100%) rename {sensor-raspi => hedgehog-raspi}/rootfs/etc/initramfs-tools/hooks/rpi-resizerootfs (100%) rename {sensor-raspi => hedgehog-raspi}/rootfs/etc/initramfs-tools/scripts/local-bottom/rpi-resizerootfs (100%) rename {sensor-raspi => hedgehog-raspi}/rootfs/etc/network/interfaces.d/eth0 (100%) rename {sensor-raspi => hedgehog-raspi}/rootfs/etc/network/interfaces.d/wlan0 (100%) rename {sensor-raspi => hedgehog-raspi}/rootfs/etc/systemd/system/rpi-generate-ssh-host-keys.service (100%) rename {sensor-raspi => hedgehog-raspi}/rootfs/etc/systemd/system/rpi-reconfigure-raspi-firmware.service (100%) rename {sensor-raspi => hedgehog-raspi}/rootfs/etc/systemd/system/rpi-set-sysconf.service (100%) rename {sensor-raspi => hedgehog-raspi}/rootfs/etc/wpa_supplicant/wpa_supplicant.conf (100%) rename {sensor-raspi => hedgehog-raspi}/rootfs/usr/local/sbin/rpi-set-sysconf (100%) rename {sensor-raspi => hedgehog-raspi}/sensor_install.sh (99%) rename {sensor-raspi => hedgehog-raspi}/vagrant/Vagrantfile (100%) rename {sensor-raspi => hedgehog-raspi}/vagrant/ssh_config (100%) diff --git a/.dockerignore b/.dockerignore index b63f0cdfa..4172fe569 100644 --- a/.dockerignore +++ b/.dockerignore @@ -28,8 +28,8 @@ arkime-logs arkime-raw kubernetes malcolm-iso -sensor-iso -sensor-raspi +hedgehog-iso +hedgehog-raspi nginx/nginx_ldap*.conf pcap _site diff --git a/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml b/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml similarity index 87% rename from .github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml rename to .github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml index 14fe12261..76361ae13 100644 --- a/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml +++ b/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml @@ -1,4 +1,4 @@ -name: sensor-iso-build-docker-wrap-push-ghcr +name: hedgehog-iso-build-docker-wrap-push-ghcr on: push: @@ -6,10 +6,10 @@ on: - main - development paths: - - 'sensor-iso/**' + - 'hedgehog-iso/**' - 'shared/bin/*' - '.trigger_iso_workflow_build' - - '.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml' + - '.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml' workflow_dispatch: repository_dispatch: @@ -93,13 +93,13 @@ jobs: - name: Build image run: | - cp -r ./shared ./docs ./_config.yml ./_includes ./_layouts ./Gemfile ./README.md ./malcolm-iso/htpdate ./sensor-iso - cp ./scripts/malcolm_utils.py ./sensor-iso/shared/bin/ - cp ./scripts/documentation_build.sh ./sensor-iso/docs/ - cp -r ./arkime/patch ./sensor-iso/shared/arkime_patch - mkdir -p ./sensor-iso/suricata - cp -r ./suricata/rules-default ./sensor-iso/suricata/ - pushd ./sensor-iso + cp -r ./shared ./docs ./_config.yml ./_includes ./_layouts ./Gemfile ./README.md ./malcolm-iso/htpdate ./hedgehog-iso + cp ./scripts/malcolm_utils.py ./hedgehog-iso/shared/bin/ + cp ./scripts/documentation_build.sh ./hedgehog-iso/docs/ + cp -r ./arkime/patch ./hedgehog-iso/shared/arkime_patch + mkdir -p ./hedgehog-iso/suricata + cp -r ./suricata/rules-default ./hedgehog-iso/suricata/ + pushd ./hedgehog-iso echo "${{ steps.extract_malcolm_version.outputs.mversion }}" > ./shared/version.txt echo "${{ secrets.MAXMIND_GEOIP_DB_LICENSE_KEY }}" > ./shared/maxmind_license.txt echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" > ./shared/environment.chroot @@ -115,7 +115,7 @@ jobs: uses: aquasecurity/trivy-action@master with: scan-type: 'fs' - scan-ref: './sensor-iso' + scan-ref: './hedgehog-iso' format: 'sarif' output: 'trivy-results.sarif' severity: 'HIGH,CRITICAL' @@ -140,6 +140,6 @@ jobs: name: Build and push ISO image uses: docker/build-push-action@v5 with: - context: ./sensor-iso + context: ./hedgehog-iso push: true tags: ghcr.io/${{ github.repository_owner }}/malcolm/hedgehog:${{ steps.extract_branch.outputs.branch }} diff --git a/.github/workflows/sensor-raspi-build-docker-wrap-push-ghcr.yml b/.github/workflows/hedgehog-raspi-build-docker-wrap-push-ghcr.yml similarity index 96% rename from .github/workflows/sensor-raspi-build-docker-wrap-push-ghcr.yml rename to .github/workflows/hedgehog-raspi-build-docker-wrap-push-ghcr.yml index 0d5faee04..952e850ab 100644 --- a/.github/workflows/sensor-raspi-build-docker-wrap-push-ghcr.yml +++ b/.github/workflows/hedgehog-raspi-build-docker-wrap-push-ghcr.yml @@ -1,4 +1,4 @@ -name: sensor-raspi-build-docker-wrap-push-ghcr +name: hedgehog-raspi-build-docker-wrap-push-ghcr on: # push: @@ -76,7 +76,7 @@ jobs: - name: Build image run: | - pushd ./sensor-raspi + pushd ./hedgehog-raspi mkdir -p ./shared echo "${{ steps.extract_malcolm_version.outputs.mversion }}" > ./shared/version.txt echo "${{ secrets.MAXMIND_GEOIP_DB_LICENSE_KEY }}" > ./shared/maxmind_license.txt @@ -97,6 +97,6 @@ jobs: name: Build and push IMG image uses: docker/build-push-action@v5 with: - context: ./sensor-raspi + context: ./hedgehog-raspi push: true tags: ghcr.io/${{ github.repository_owner }}/malcolm/hedgehog-raspi:${{ steps.extract_branch.outputs.branch }} diff --git a/.gitignore b/.gitignore index eadc2e2ff..1b3fe1317 100644 --- a/.gitignore +++ b/.gitignore @@ -37,13 +37,13 @@ malcolm_netbox_backup_*.gz *-build.log Gemfile.lock _site -sensor-iso/_config.yml -sensor-iso/_includes -sensor-iso/_layouts -sensor-iso/_site -sensor-iso/docs -sensor-iso/Gemfile -sensor-iso/README.md +hedgehog-iso/_config.yml +hedgehog-iso/_includes +hedgehog-iso/_layouts +hedgehog-iso/_site +hedgehog-iso/docs +hedgehog-iso/Gemfile +hedgehog-iso/README.md # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/_config.yml b/_config.yml index 0baeaec2c..cc752665c 100644 --- a/_config.yml +++ b/_config.yml @@ -84,7 +84,7 @@ exclude: - pcap-capture - pcap-monitor - scripts - - sensor-iso + - hedgehog-iso - shared - suricata - suricata-logs diff --git a/docs/development.md b/docs/development.md index b59bc3b49..c8efe2d94 100644 --- a/docs/development.md +++ b/docs/development.md @@ -16,6 +16,8 @@ Checking out the [Malcolm source code]({{ site.github.repository_url }}/tree/{{ * `file-monitor` - code and configuration for the `file-monitor` container that can scan files extracted by Zeek * `file-upload` - code and configuration for the `upload` container that serves a web browser-based upload form for uploading PCAP files and Zeek logs, and serves an SFTP share as an alternate upload * `freq-server` - code and configuration for the `freq` container used for calculating entropy of strings +* `hedgehog-iso` - code and configuration for building a [Hedgehog Linux](live-analysis.md#Hedgehog) ISO +* `hedgehog-raspi` - code and configuration for building a [Hedgehog Linux](live-analysis.md#Hedgehog) [Raspberry Pi image](hedgehog-raspi-build.md#HedgehogRaspiBuild) * `htadmin` - configuration for the `htadmin` user account management container * `logstash` - code and configuration for the `logstash` container that parses Zeek logs and forwards them to the `opensearch` container * `malcolm-iso` - code and configuration for building an [installer ISO](malcolm-iso.md#ISO) for a minimal Debian-based Linux installation for running Malcolm @@ -27,7 +29,6 @@ Checking out the [Malcolm source code]({{ site.github.repository_url }}/tree/{{ * `pcap-capture` - code and configuration for the `pcap-capture` container that can capture network traffic * `pcap-monitor` - code and configuration for the `pcap-monitor` container that watches for new or uploaded PCAP files and notifies the other services to process them * `scripts` - control scripts for starting, stopping, restarting, etc., Malcolm -* `sensor-iso` - code and configuration for building a [Hedgehog Linux](live-analysis.md#Hedgehog) ISO * `shared` - miscellaneous code used by various Malcolm components * `suricata` - code and configuration for the `suricata` container that handles PCAP processing using Suricata * `suricata-logs` - an initially empty directory for Suricata logs to be uploaded, processed, and stored diff --git a/docs/hedgehog-iso-build.md b/docs/hedgehog-iso-build.md index 94a8877fd..4de1fe858 100644 --- a/docs/hedgehog-iso-build.md +++ b/docs/hedgehog-iso-build.md @@ -18,7 +18,7 @@ The build should work with either the [VirtualBox](https://www.virtualbox.org/) To perform a clean build the Hedgehog Linux installer ISO, navigate to your local [Malcolm]({{ site.github.repository_url }}/) working copy and run: ``` -$ ./sensor-iso/build_via_vagrant.sh -f +$ ./hedgehog-iso/build_via_vagrant.sh -f … Starting build machine... Bringing machine 'default' up with 'virtualbox' provider... @@ -33,4 +33,4 @@ Finished, created "/sensor-build/hedgehog-24.03.0.iso" … ``` -Alternately, if you have forked Malcolm on GitHub, [workflow files]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/.github/workflows/) are provided which contain instructions for GitHub to build the docker images and Hedgehog and [Malcolm]({{ site.github.repository_url }}) installer ISOs, specifically [`sensor-iso-build-docker-wrap-push-ghcr.yml`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml) for the Hedgehog ISO. The resulting ISO file is wrapped in a Docker image that provides an HTTP server from which the ISO may be downloaded. \ No newline at end of file +Alternately, if you have forked Malcolm on GitHub, [workflow files]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/.github/workflows/) are provided which contain instructions for GitHub to build the docker images and Hedgehog and [Malcolm]({{ site.github.repository_url }}) installer ISOs, specifically [`hedgehog-iso-build-docker-wrap-push-ghcr.yml`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml) for the Hedgehog ISO. The resulting ISO file is wrapped in a Docker image that provides an HTTP server from which the ISO may be downloaded. \ No newline at end of file diff --git a/docs/hedgehog-raspi-build.md b/docs/hedgehog-raspi-build.md index 82e4dc188..6d1f0cf8d 100644 --- a/docs/hedgehog-raspi-build.md +++ b/docs/hedgehog-raspi-build.md @@ -1,6 +1,6 @@ # Appendix B - Generating a Raspberry Pi Image -Hedgehog Linux can [also be built]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/sensor-raspi) for the Raspberry Pi platform, although this capability is still considered experimental. +Hedgehog Linux can [also be built]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/hedgehog-raspi) for the Raspberry Pi platform, although this capability is still considered experimental. * [Building the Image](#HedgehogRaspiBuild) * [Writing the Image to Flash Media](#HedgehogRaspiBurn) @@ -27,7 +27,7 @@ The build should work with either the [VirtualBox](https://www.virtualbox.org/) To perform a clean build the Hedgehog Linux Raspberry Pi image, navigate to your local [Malcolm]({{ site.github.repository_url }}/) working copy and run: ``` -$ ./sensor-raspi/build_via_vagrant.sh -f -z +$ ./hedgehog-raspi/build_via_vagrant.sh -f -z … Starting build machine... Bringing machine 'vagrant-hedgehog-raspi-build' up with 'virtualbox' provider... diff --git a/docs/hedgehog-upgrade.md b/docs/hedgehog-upgrade.md index 37d8a1a53..3b3cec849 100644 --- a/docs/hedgehog-upgrade.md +++ b/docs/hedgehog-upgrade.md @@ -47,9 +47,9 @@ deb https://XXXXXX:443/debian buster-backports main contrib non-free 5. Update underlying system packages with `apt-get` - `apt-get update && apt-get dist-upgrade` -6. If there were [new system deb packages added]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/sensor-iso/config/package-lists) to this release of Hedgehog Linux (you might have to [manually compare]({{ site.github.repository_url }}/commits/main/sensor-iso/config/package-lists) on GitHub), install them. If you're not sure, of course, you could just install everything, like this (although you may have to tweak some version numbers or something if the base distribution of your Hedgehog branch is different than `main`; in this example I'm not jumping between Debian releases, just upgrading within a release): +6. If there were [new system deb packages added]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/hedgehog-iso/config/package-lists) to this release of Hedgehog Linux (you might have to [manually compare]({{ site.github.repository_url }}/commits/main/hedgehog-iso/config/package-lists) on GitHub), install them. If you're not sure, of course, you could just install everything, like this (although you may have to tweak some version numbers or something if the base distribution of your Hedgehog branch is different than `main`; in this example I'm not jumping between Debian releases, just upgrading within a release): ``` -$ for LIST in apps desktopmanager net system; do curl -L -J -O {{ site.github.repository_url }}/main/sensor-iso/config/package-lists/$LIST.list.chroot; done +$ for LIST in apps desktopmanager net system; do curl -L -J -O {{ site.github.repository_url }}/main/hedgehog-iso/config/package-lists/$LIST.list.chroot; done ... $ apt-get install $(cat *.list.chroot) ``` @@ -59,7 +59,7 @@ $ apt-get install $(cat *.list.chroot) * `python3 -m pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1 | xargs -r -n1 python3 -m pip install -U` - if this fails for some reason, you may need to reinstall pip first with `python3 -m pip install --force -U pip` - some *very* old builds of Hedgehog Linux had separate Python 3.5 and 3.7 installations: in this case, you'd need to do this for both `python3 -m pip` and `python3.7 -m pip` (or whatever `python3.x` you have) - * If there were [new python packages](https://raw.githubusercontent.com/{{ site.github.repository_nwo }}/master/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot) added to this release of Hedgehog Linux (you might have to [manually compare]({{ site.github.repository_url }}/blame/main/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot) on GitHub), install them. If you are using a PyPI mirror, replace `XXXXXX` here with your mirror's IP. The `colorama` package is used here as an example, your package list might vary. + * If there were [new python packages](https://raw.githubusercontent.com/{{ site.github.repository_nwo }}/master/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot) added to this release of Hedgehog Linux (you might have to [manually compare]({{ site.github.repository_url }}/blame/main/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot) on GitHub), install them. If you are using a PyPI mirror, replace `XXXXXX` here with your mirror's IP. The `colorama` package is used here as an example, your package list might vary. - `python3 -m pip install --no-compile --no-cache-dir --force-reinstall --upgrade --index-url=https://XXXXXX:443/pypi/simple --trusted-host=XXXXXX:443 colorama` 8. Okay, **now** things start to get a little bit ugly. You're going to need access to the ISO of the release of Hedgehog Linux you're upgrading to, as we're going to grab some packages off of it. On another Linux system, [build it](hedgehog-iso-build.md#HedgehogISOBuild). diff --git a/docs/hedgehog.md b/docs/hedgehog.md index f3399d5e3..94880fa51 100644 --- a/docs/hedgehog.md +++ b/docs/hedgehog.md @@ -11,7 +11,7 @@ Hedgehog Linux is a Debian-based operating system built to * detect file transfers in network traffic and extract and scan those files for threats * generate and forward Zeek logs, Arkime sessions, and other information to [Malcolm]({{ site.github.repository_url }}) -![sensor-iso-build-docker-wrap-push-ghcr]({{ site.github.repository_url }}/workflows/sensor-iso-build-docker-wrap-push-ghcr/badge.svg) +![hedgehog-iso-build-docker-wrap-push-ghcr]({{ site.github.repository_url }}/workflows/hedgehog-iso-build-docker-wrap-push-ghcr/badge.svg) * [Sensor installation](hedgehog-installation.md#HedgehogInstallation) diff --git a/docs/malcolm-hedgehog-e2e-iso-install.md b/docs/malcolm-hedgehog-e2e-iso-install.md index 666bfc743..71a303031 100644 --- a/docs/malcolm-hedgehog-e2e-iso-install.md +++ b/docs/malcolm-hedgehog-e2e-iso-install.md @@ -463,7 +463,7 @@ To specify which files should be extracted, specify the Zeek file carving mode: ![Zeek file carving mode](./images/hedgehog/images/zeek_file_carve_mode.png) -If unsure what mode to choose, both **mapped (except common plain text files)** (to carve and scan almost all files) and **interesting** (to only carve and scan files with [mime types of common attack vectors]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/sensor-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek)) are probably good choices. +If unsure what mode to choose, both **mapped (except common plain text files)** (to carve and scan almost all files) and **interesting** (to only carve and scan files with [mime types of common attack vectors]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/hedgehog-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek)) are probably good choices. Next, specify which carved files to preserve (saved on the sensor under `/capture/bro/capture/extract_files/quarantine` by default). In order to not consume all the sensor's available storage space, the oldest preserved files will be pruned along with the oldest Zeek logs as described below with **AUTOSTART_PRUNE_ZEEK** in the [autostart services](#HedgehogConfigAutostart) section. diff --git a/sensor-iso/.dockerignore b/hedgehog-iso/.dockerignore similarity index 100% rename from sensor-iso/.dockerignore rename to hedgehog-iso/.dockerignore diff --git a/sensor-iso/.gitignore b/hedgehog-iso/.gitignore similarity index 100% rename from sensor-iso/.gitignore rename to hedgehog-iso/.gitignore diff --git a/sensor-iso/Dockerfile b/hedgehog-iso/Dockerfile similarity index 95% rename from sensor-iso/Dockerfile rename to hedgehog-iso/Dockerfile index 774c62213..c7a9970b2 100644 --- a/sensor-iso/Dockerfile +++ b/hedgehog-iso/Dockerfile @@ -5,7 +5,7 @@ FROM ghcr.io/mmguero/qemu-live-iso:latest LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' -LABEL org.opencontainers.image.documentation='https://github.com/idaholab/Malcolm/blob/main/sensor-iso/README.md' +LABEL org.opencontainers.image.documentation='https://github.com/idaholab/Malcolm/blob/main/hedgehog-iso/README.md' LABEL org.opencontainers.image.source='https://github.com/idaholab/Malcolm' LABEL org.opencontainers.image.vendor='Idaho National Laboratory' LABEL org.opencontainers.image.title='ghcr.io/idaholab/malcolm/hedgehog' diff --git a/sensor-iso/arkime/Dockerfile b/hedgehog-iso/arkime/Dockerfile similarity index 100% rename from sensor-iso/arkime/Dockerfile rename to hedgehog-iso/arkime/Dockerfile diff --git a/sensor-iso/arkime/build-arkime-deb.sh b/hedgehog-iso/arkime/build-arkime-deb.sh similarity index 100% rename from sensor-iso/arkime/build-arkime-deb.sh rename to hedgehog-iso/arkime/build-arkime-deb.sh diff --git a/sensor-iso/arkime/build-docker-image.sh b/hedgehog-iso/arkime/build-docker-image.sh similarity index 100% rename from sensor-iso/arkime/build-docker-image.sh rename to hedgehog-iso/arkime/build-docker-image.sh diff --git a/sensor-iso/arkime/etc/.gitignore b/hedgehog-iso/arkime/etc/.gitignore similarity index 100% rename from sensor-iso/arkime/etc/.gitignore rename to hedgehog-iso/arkime/etc/.gitignore diff --git a/sensor-iso/build.sh b/hedgehog-iso/build.sh similarity index 100% rename from sensor-iso/build.sh rename to hedgehog-iso/build.sh diff --git a/sensor-iso/build_via_vagrant.sh b/hedgehog-iso/build_via_vagrant.sh similarity index 100% rename from sensor-iso/build_via_vagrant.sh rename to hedgehog-iso/build_via_vagrant.sh diff --git a/sensor-iso/config/archives/beats.key.binary b/hedgehog-iso/config/archives/beats.key.binary similarity index 100% rename from sensor-iso/config/archives/beats.key.binary rename to hedgehog-iso/config/archives/beats.key.binary diff --git a/sensor-iso/config/archives/beats.key.chroot b/hedgehog-iso/config/archives/beats.key.chroot similarity index 100% rename from sensor-iso/config/archives/beats.key.chroot rename to hedgehog-iso/config/archives/beats.key.chroot diff --git a/sensor-iso/config/archives/beats.list.binary b/hedgehog-iso/config/archives/beats.list.binary similarity index 100% rename from sensor-iso/config/archives/beats.list.binary rename to hedgehog-iso/config/archives/beats.list.binary diff --git a/sensor-iso/config/archives/beats.list.chroot b/hedgehog-iso/config/archives/beats.list.chroot similarity index 100% rename from sensor-iso/config/archives/beats.list.chroot rename to hedgehog-iso/config/archives/beats.list.chroot diff --git a/sensor-iso/config/archives/fluentbit.key.binary b/hedgehog-iso/config/archives/fluentbit.key.binary similarity index 100% rename from sensor-iso/config/archives/fluentbit.key.binary rename to hedgehog-iso/config/archives/fluentbit.key.binary diff --git a/sensor-iso/config/archives/fluentbit.key.chroot b/hedgehog-iso/config/archives/fluentbit.key.chroot similarity index 100% rename from sensor-iso/config/archives/fluentbit.key.chroot rename to hedgehog-iso/config/archives/fluentbit.key.chroot diff --git a/sensor-iso/config/archives/fluentbit.list.binary b/hedgehog-iso/config/archives/fluentbit.list.binary similarity index 100% rename from sensor-iso/config/archives/fluentbit.list.binary rename to hedgehog-iso/config/archives/fluentbit.list.binary diff --git a/sensor-iso/config/archives/fluentbit.list.chroot b/hedgehog-iso/config/archives/fluentbit.list.chroot similarity index 100% rename from sensor-iso/config/archives/fluentbit.list.chroot rename to hedgehog-iso/config/archives/fluentbit.list.chroot diff --git a/sensor-iso/config/archives/mozilla.key.binary b/hedgehog-iso/config/archives/mozilla.key.binary similarity index 100% rename from sensor-iso/config/archives/mozilla.key.binary rename to hedgehog-iso/config/archives/mozilla.key.binary diff --git a/sensor-iso/config/archives/mozilla.key.chroot b/hedgehog-iso/config/archives/mozilla.key.chroot similarity index 100% rename from sensor-iso/config/archives/mozilla.key.chroot rename to hedgehog-iso/config/archives/mozilla.key.chroot diff --git a/sensor-iso/config/archives/mozilla.list.binary b/hedgehog-iso/config/archives/mozilla.list.binary similarity index 100% rename from sensor-iso/config/archives/mozilla.list.binary rename to hedgehog-iso/config/archives/mozilla.list.binary diff --git a/sensor-iso/config/archives/mozilla.list.chroot b/hedgehog-iso/config/archives/mozilla.list.chroot similarity index 100% rename from sensor-iso/config/archives/mozilla.list.chroot rename to hedgehog-iso/config/archives/mozilla.list.chroot diff --git a/sensor-iso/config/archives/mozilla.pref.binary b/hedgehog-iso/config/archives/mozilla.pref.binary similarity index 100% rename from sensor-iso/config/archives/mozilla.pref.binary rename to hedgehog-iso/config/archives/mozilla.pref.binary diff --git a/sensor-iso/config/archives/mozilla.pref.chroot b/hedgehog-iso/config/archives/mozilla.pref.chroot similarity index 100% rename from sensor-iso/config/archives/mozilla.pref.chroot rename to hedgehog-iso/config/archives/mozilla.pref.chroot diff --git a/sensor-iso/config/bootloaders/extlinux/extlinux.conf b/hedgehog-iso/config/bootloaders/extlinux/extlinux.conf similarity index 100% rename from sensor-iso/config/bootloaders/extlinux/extlinux.conf rename to hedgehog-iso/config/bootloaders/extlinux/extlinux.conf diff --git a/sensor-iso/config/bootloaders/grub-legacy/menu.lst b/hedgehog-iso/config/bootloaders/grub-legacy/menu.lst similarity index 100% rename from sensor-iso/config/bootloaders/grub-legacy/menu.lst rename to hedgehog-iso/config/bootloaders/grub-legacy/menu.lst diff --git a/sensor-iso/config/bootloaders/grub-legacy/splash.xpm.gz b/hedgehog-iso/config/bootloaders/grub-legacy/splash.xpm.gz similarity index 100% rename from sensor-iso/config/bootloaders/grub-legacy/splash.xpm.gz rename to hedgehog-iso/config/bootloaders/grub-legacy/splash.xpm.gz diff --git a/sensor-iso/config/bootloaders/grub-pc/config.cfg b/hedgehog-iso/config/bootloaders/grub-pc/config.cfg similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/config.cfg rename to hedgehog-iso/config/bootloaders/grub-pc/config.cfg diff --git a/sensor-iso/config/bootloaders/grub-pc/dejavu-bold-14.pf2 b/hedgehog-iso/config/bootloaders/grub-pc/dejavu-bold-14.pf2 similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/dejavu-bold-14.pf2 rename to hedgehog-iso/config/bootloaders/grub-pc/dejavu-bold-14.pf2 diff --git a/sensor-iso/config/bootloaders/grub-pc/dejavu-bold-16.pf2 b/hedgehog-iso/config/bootloaders/grub-pc/dejavu-bold-16.pf2 similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/dejavu-bold-16.pf2 rename to hedgehog-iso/config/bootloaders/grub-pc/dejavu-bold-16.pf2 diff --git a/sensor-iso/config/bootloaders/grub-pc/grub.cfg b/hedgehog-iso/config/bootloaders/grub-pc/grub.cfg similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/grub.cfg rename to hedgehog-iso/config/bootloaders/grub-pc/grub.cfg diff --git a/sensor-iso/config/bootloaders/grub-pc/hedgehog-color.png b/hedgehog-iso/config/bootloaders/grub-pc/hedgehog-color.png similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/hedgehog-color.png rename to hedgehog-iso/config/bootloaders/grub-pc/hedgehog-color.png diff --git a/sensor-iso/config/bootloaders/grub-pc/install_start_text.cfg b/hedgehog-iso/config/bootloaders/grub-pc/install_start_text.cfg similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/install_start_text.cfg rename to hedgehog-iso/config/bootloaders/grub-pc/install_start_text.cfg diff --git a/sensor-iso/config/bootloaders/grub-pc/install_text.cfg b/hedgehog-iso/config/bootloaders/grub-pc/install_text.cfg similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/install_text.cfg rename to hedgehog-iso/config/bootloaders/grub-pc/install_text.cfg diff --git a/sensor-iso/config/bootloaders/grub-pc/live-theme/theme.txt b/hedgehog-iso/config/bootloaders/grub-pc/live-theme/theme.txt similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/live-theme/theme.txt rename to hedgehog-iso/config/bootloaders/grub-pc/live-theme/theme.txt diff --git a/sensor-iso/config/bootloaders/grub-pc/memtest.cfg b/hedgehog-iso/config/bootloaders/grub-pc/memtest.cfg similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/memtest.cfg rename to hedgehog-iso/config/bootloaders/grub-pc/memtest.cfg diff --git a/sensor-iso/config/bootloaders/grub-pc/splash.png b/hedgehog-iso/config/bootloaders/grub-pc/splash.png similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/splash.png rename to hedgehog-iso/config/bootloaders/grub-pc/splash.png diff --git a/sensor-iso/config/bootloaders/grub-pc/theme.cfg b/hedgehog-iso/config/bootloaders/grub-pc/theme.cfg similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/theme.cfg rename to hedgehog-iso/config/bootloaders/grub-pc/theme.cfg diff --git a/sensor-iso/config/bootloaders/isolinux/isolinux.cfg b/hedgehog-iso/config/bootloaders/isolinux/isolinux.cfg similarity index 100% rename from sensor-iso/config/bootloaders/isolinux/isolinux.cfg rename to hedgehog-iso/config/bootloaders/isolinux/isolinux.cfg diff --git a/sensor-iso/config/bootloaders/pxelinux/pxelinux.cfg/default b/hedgehog-iso/config/bootloaders/pxelinux/pxelinux.cfg/default similarity index 100% rename from sensor-iso/config/bootloaders/pxelinux/pxelinux.cfg/default rename to hedgehog-iso/config/bootloaders/pxelinux/pxelinux.cfg/default diff --git a/sensor-iso/config/bootloaders/syslinux/syslinux.cfg b/hedgehog-iso/config/bootloaders/syslinux/syslinux.cfg similarity index 100% rename from sensor-iso/config/bootloaders/syslinux/syslinux.cfg rename to hedgehog-iso/config/bootloaders/syslinux/syslinux.cfg diff --git a/sensor-iso/config/bootloaders/syslinux_common/install_text.cfg b/hedgehog-iso/config/bootloaders/syslinux_common/install_text.cfg similarity index 100% rename from sensor-iso/config/bootloaders/syslinux_common/install_text.cfg rename to hedgehog-iso/config/bootloaders/syslinux_common/install_text.cfg diff --git a/sensor-iso/config/bootloaders/syslinux_common/live.cfg.in b/hedgehog-iso/config/bootloaders/syslinux_common/live.cfg.in similarity index 100% rename from sensor-iso/config/bootloaders/syslinux_common/live.cfg.in rename to hedgehog-iso/config/bootloaders/syslinux_common/live.cfg.in diff --git a/sensor-iso/config/bootloaders/syslinux_common/memtest.cfg b/hedgehog-iso/config/bootloaders/syslinux_common/memtest.cfg similarity index 100% rename from sensor-iso/config/bootloaders/syslinux_common/memtest.cfg rename to hedgehog-iso/config/bootloaders/syslinux_common/memtest.cfg diff --git a/sensor-iso/config/bootloaders/syslinux_common/menu.cfg b/hedgehog-iso/config/bootloaders/syslinux_common/menu.cfg similarity index 100% rename from sensor-iso/config/bootloaders/syslinux_common/menu.cfg rename to hedgehog-iso/config/bootloaders/syslinux_common/menu.cfg diff --git a/sensor-iso/config/bootloaders/syslinux_common/splash.svg b/hedgehog-iso/config/bootloaders/syslinux_common/splash.svg similarity index 100% rename from sensor-iso/config/bootloaders/syslinux_common/splash.svg rename to hedgehog-iso/config/bootloaders/syslinux_common/splash.svg diff --git a/sensor-iso/config/bootloaders/syslinux_common/stdmenu.cfg b/hedgehog-iso/config/bootloaders/syslinux_common/stdmenu.cfg similarity index 100% rename from sensor-iso/config/bootloaders/syslinux_common/stdmenu.cfg rename to hedgehog-iso/config/bootloaders/syslinux_common/stdmenu.cfg diff --git a/sensor-iso/config/bootloaders/syslinux_common/utilities.cfg b/hedgehog-iso/config/bootloaders/syslinux_common/utilities.cfg similarity index 100% rename from sensor-iso/config/bootloaders/syslinux_common/utilities.cfg rename to hedgehog-iso/config/bootloaders/syslinux_common/utilities.cfg diff --git a/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot similarity index 100% rename from sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot rename to hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot diff --git a/sensor-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot b/hedgehog-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot similarity index 100% rename from sensor-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot rename to hedgehog-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot diff --git a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot similarity index 100% rename from sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot rename to hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot diff --git a/sensor-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot b/hedgehog-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot similarity index 100% rename from sensor-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot rename to hedgehog-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot diff --git a/sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot b/hedgehog-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot similarity index 100% rename from sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot rename to hedgehog-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot diff --git a/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot b/hedgehog-iso/config/hooks/normal/0991-security-performance.hook.chroot similarity index 100% rename from sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot rename to hedgehog-iso/config/hooks/normal/0991-security-performance.hook.chroot diff --git a/sensor-iso/config/hooks/normal/0992-localepurge.hook.chroot b/hedgehog-iso/config/hooks/normal/0992-localepurge.hook.chroot similarity index 100% rename from sensor-iso/config/hooks/normal/0992-localepurge.hook.chroot rename to hedgehog-iso/config/hooks/normal/0992-localepurge.hook.chroot diff --git a/sensor-iso/config/hooks/normal/0992-login.hook.chroot b/hedgehog-iso/config/hooks/normal/0992-login.hook.chroot similarity index 100% rename from sensor-iso/config/hooks/normal/0992-login.hook.chroot rename to hedgehog-iso/config/hooks/normal/0992-login.hook.chroot diff --git a/sensor-iso/config/includes.binary/install/preseed_base.cfg b/hedgehog-iso/config/includes.binary/install/preseed_base.cfg similarity index 100% rename from sensor-iso/config/includes.binary/install/preseed_base.cfg rename to hedgehog-iso/config/includes.binary/install/preseed_base.cfg diff --git a/sensor-iso/config/includes.binary/install/preseed_multipar.cfg b/hedgehog-iso/config/includes.binary/install/preseed_multipar.cfg similarity index 100% rename from sensor-iso/config/includes.binary/install/preseed_multipar.cfg rename to hedgehog-iso/config/includes.binary/install/preseed_multipar.cfg diff --git a/sensor-iso/config/includes.binary/install/preseed_vmware.cfg b/hedgehog-iso/config/includes.binary/install/preseed_vmware.cfg similarity index 100% rename from sensor-iso/config/includes.binary/install/preseed_vmware.cfg rename to hedgehog-iso/config/includes.binary/install/preseed_vmware.cfg diff --git a/sensor-iso/config/includes.binary/live/filesystem.packages-remove b/hedgehog-iso/config/includes.binary/live/filesystem.packages-remove similarity index 100% rename from sensor-iso/config/includes.binary/live/filesystem.packages-remove rename to hedgehog-iso/config/includes.binary/live/filesystem.packages-remove diff --git a/sensor-iso/config/includes.chroot/etc/audit/rules.d/audit.rules b/hedgehog-iso/config/includes.chroot/etc/audit/rules.d/audit.rules similarity index 100% rename from sensor-iso/config/includes.chroot/etc/audit/rules.d/audit.rules rename to hedgehog-iso/config/includes.chroot/etc/audit/rules.d/audit.rules diff --git a/sensor-iso/config/includes.chroot/etc/bash.bash_aliases b/hedgehog-iso/config/includes.chroot/etc/bash.bash_aliases similarity index 100% rename from sensor-iso/config/includes.chroot/etc/bash.bash_aliases rename to hedgehog-iso/config/includes.chroot/etc/bash.bash_aliases diff --git a/sensor-iso/config/includes.chroot/etc/bash.bash_functions b/hedgehog-iso/config/includes.chroot/etc/bash.bash_functions similarity index 100% rename from sensor-iso/config/includes.chroot/etc/bash.bash_functions rename to hedgehog-iso/config/includes.chroot/etc/bash.bash_functions diff --git a/sensor-iso/config/includes.chroot/etc/installer b/hedgehog-iso/config/includes.chroot/etc/installer similarity index 100% rename from sensor-iso/config/includes.chroot/etc/installer rename to hedgehog-iso/config/includes.chroot/etc/installer diff --git a/sensor-iso/config/includes.chroot/etc/lightdm/slick-greeter.conf b/hedgehog-iso/config/includes.chroot/etc/lightdm/slick-greeter.conf similarity index 100% rename from sensor-iso/config/includes.chroot/etc/lightdm/slick-greeter.conf rename to hedgehog-iso/config/includes.chroot/etc/lightdm/slick-greeter.conf diff --git a/sensor-iso/config/includes.chroot/etc/live/config.conf b/hedgehog-iso/config/includes.chroot/etc/live/config.conf similarity index 100% rename from sensor-iso/config/includes.chroot/etc/live/config.conf rename to hedgehog-iso/config/includes.chroot/etc/live/config.conf diff --git a/sensor-iso/config/includes.chroot/etc/login.defs b/hedgehog-iso/config/includes.chroot/etc/login.defs similarity index 100% rename from sensor-iso/config/includes.chroot/etc/login.defs rename to hedgehog-iso/config/includes.chroot/etc/login.defs diff --git a/sensor-iso/config/includes.chroot/etc/security/limits.d/limits.conf b/hedgehog-iso/config/includes.chroot/etc/security/limits.d/limits.conf similarity index 100% rename from sensor-iso/config/includes.chroot/etc/security/limits.d/limits.conf rename to hedgehog-iso/config/includes.chroot/etc/security/limits.d/limits.conf diff --git a/sensor-iso/config/includes.chroot/etc/skel/.bashrc b/hedgehog-iso/config/includes.chroot/etc/skel/.bashrc similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.bashrc rename to hedgehog-iso/config/includes.chroot/etc/skel/.bashrc diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/autostart/capture-format-wait.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/autostart/capture-format-wait.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/autostart/capture-format-wait.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/autostart/capture-format-wait.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/autostart/dod-login-banner.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/autostart/dod-login-banner.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/autostart/dod-login-banner.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/autostart/dod-login-banner.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/autostart/hedgehog-kiosk.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/autostart/hedgehog-kiosk.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/autostart/hedgehog-kiosk.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/autostart/hedgehog-kiosk.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/autostart/set-dconf-screen-lock-defaults.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/autostart/set-dconf-screen-lock-defaults.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/autostart/set-dconf-screen-lock-defaults.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/autostart/set-dconf-screen-lock-defaults.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/dconf/user b/hedgehog-iso/config/includes.chroot/etc/skel/.config/dconf/user similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/dconf/user rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/dconf/user diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/bookmarks b/hedgehog-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/bookmarks similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/bookmarks rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/bookmarks diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/settings.ini b/hedgehog-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/settings.ini similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/settings.ini rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/settings.ini diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/light-locker-dconf-defaults.conf b/hedgehog-iso/config/includes.chroot/etc/skel/.config/light-locker-dconf-defaults.conf similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/light-locker-dconf-defaults.conf rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/light-locker-dconf-defaults.conf diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/mimeapps.list b/hedgehog-iso/config/includes.chroot/etc/skel/.config/mimeapps.list similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/mimeapps.list rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/mimeapps.list diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/helpers.rc b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/helpers.rc similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/helpers.rc rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/helpers.rc diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-15/16346759461.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-15/16346759461.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-15/16346759461.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-15/16346759461.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-16/16346759562.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-16/16346759562.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-16/16346759562.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-16/16346759562.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-17/16346759653.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-17/16346759653.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-17/16346759653.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-17/16346759653.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-18/16346759724.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-18/16346759724.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-18/16346759724.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-18/16346759724.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-19/16346759855.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-19/16346759855.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-19/16346759855.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-19/16346759855.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-20/16346759956.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-20/16346759956.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-20/16346759956.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-20/16346759956.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-21/16346760027.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-21/16346760027.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-21/16346760027.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-21/16346760027.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-22/16346760098.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-22/16346760098.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-22/16346760098.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-22/16346760098.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/terminal/terminalrc b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/terminal/terminalrc similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/terminal/terminalrc rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/terminal/terminalrc diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-desktop.xml b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-desktop.xml similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-desktop.xml rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-desktop.xml diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-power-manager.xml b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-power-manager.xml similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-power-manager.xml rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-power-manager.xml diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-session.xml b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-session.xml similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-session.xml rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-session.xml diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfwm4.xml b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfwm4.xml similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfwm4.xml rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfwm4.xml diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xsettings.xml b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xsettings.xml similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xsettings.xml rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xsettings.xml diff --git a/sensor-iso/config/includes.chroot/etc/skel/.hushlogin b/hedgehog-iso/config/includes.chroot/etc/skel/.hushlogin similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.hushlogin rename to hedgehog-iso/config/includes.chroot/etc/skel/.hushlogin diff --git a/sensor-iso/config/includes.chroot/etc/skel/.local/share/xfce4/helpers/custom-TerminalEmulator.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.local/share/xfce4/helpers/custom-TerminalEmulator.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.local/share/xfce4/helpers/custom-TerminalEmulator.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.local/share/xfce4/helpers/custom-TerminalEmulator.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.selected_editor b/hedgehog-iso/config/includes.chroot/etc/skel/.selected_editor similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.selected_editor rename to hedgehog-iso/config/includes.chroot/etc/skel/.selected_editor diff --git a/sensor-iso/config/includes.chroot/etc/skel/.tmux.conf b/hedgehog-iso/config/includes.chroot/etc/skel/.tmux.conf similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.tmux.conf rename to hedgehog-iso/config/includes.chroot/etc/skel/.tmux.conf diff --git a/sensor-iso/config/includes.chroot/etc/skel/.vimrc b/hedgehog-iso/config/includes.chroot/etc/skel/.vimrc similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.vimrc rename to hedgehog-iso/config/includes.chroot/etc/skel/.vimrc diff --git a/sensor-iso/config/includes.chroot/etc/ssh/sshd_config b/hedgehog-iso/config/includes.chroot/etc/ssh/sshd_config similarity index 100% rename from sensor-iso/config/includes.chroot/etc/ssh/sshd_config rename to hedgehog-iso/config/includes.chroot/etc/ssh/sshd_config diff --git a/sensor-iso/config/includes.chroot/etc/sudoers.d/aide_integrity_check b/hedgehog-iso/config/includes.chroot/etc/sudoers.d/aide_integrity_check similarity index 100% rename from sensor-iso/config/includes.chroot/etc/sudoers.d/aide_integrity_check rename to hedgehog-iso/config/includes.chroot/etc/sudoers.d/aide_integrity_check diff --git a/sensor-iso/config/includes.chroot/etc/sudoers.d/nic_capture_setup b/hedgehog-iso/config/includes.chroot/etc/sudoers.d/nic_capture_setup similarity index 100% rename from sensor-iso/config/includes.chroot/etc/sudoers.d/nic_capture_setup rename to hedgehog-iso/config/includes.chroot/etc/sudoers.d/nic_capture_setup diff --git a/sensor-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer b/hedgehog-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer similarity index 100% rename from sensor-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer rename to hedgehog-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer diff --git a/sensor-iso/config/includes.chroot/etc/systemd/system/sensor-shutdown.service b/hedgehog-iso/config/includes.chroot/etc/systemd/system/sensor-shutdown.service similarity index 100% rename from sensor-iso/config/includes.chroot/etc/systemd/system/sensor-shutdown.service rename to hedgehog-iso/config/includes.chroot/etc/systemd/system/sensor-shutdown.service diff --git a/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor.zeek b/hedgehog-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor.zeek similarity index 100% rename from sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor.zeek rename to hedgehog-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor.zeek diff --git a/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor_params.zeek b/hedgehog-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor_params.zeek similarity index 100% rename from sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor_params.zeek rename to hedgehog-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor_params.zeek diff --git a/sensor-iso/config/includes.chroot/root/.selected_editor b/hedgehog-iso/config/includes.chroot/root/.selected_editor similarity index 100% rename from sensor-iso/config/includes.chroot/root/.selected_editor rename to hedgehog-iso/config/includes.chroot/root/.selected_editor diff --git a/sensor-iso/config/includes.chroot/root/.vimrc b/hedgehog-iso/config/includes.chroot/root/.vimrc similarity index 100% rename from sensor-iso/config/includes.chroot/root/.vimrc rename to hedgehog-iso/config/includes.chroot/root/.vimrc diff --git a/sensor-iso/config/includes.chroot/usr/local/bin/hedgehog-kiosk.sh b/hedgehog-iso/config/includes.chroot/usr/local/bin/hedgehog-kiosk.sh similarity index 100% rename from sensor-iso/config/includes.chroot/usr/local/bin/hedgehog-kiosk.sh rename to hedgehog-iso/config/includes.chroot/usr/local/bin/hedgehog-kiosk.sh diff --git a/sensor-iso/config/includes.chroot/usr/local/bin/sensor-shutdown b/hedgehog-iso/config/includes.chroot/usr/local/bin/sensor-shutdown similarity index 100% rename from sensor-iso/config/includes.chroot/usr/local/bin/sensor-shutdown rename to hedgehog-iso/config/includes.chroot/usr/local/bin/sensor-shutdown diff --git a/sensor-iso/config/includes.chroot/usr/local/etc/zeek/guess.zeek b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/guess.zeek similarity index 100% rename from sensor-iso/config/includes.chroot/usr/local/etc/zeek/guess.zeek rename to hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/guess.zeek diff --git a/sensor-iso/config/includes.chroot/usr/local/etc/zeek/guess_ics_map.txt b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/guess_ics_map.txt similarity index 100% rename from sensor-iso/config/includes.chroot/usr/local/etc/zeek/guess_ics_map.txt rename to hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/guess_ics_map.txt diff --git a/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek similarity index 100% rename from sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek rename to hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek diff --git a/sensor-iso/config/includes.chroot/usr/local/etc/zeek/login.zeek b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/login.zeek similarity index 100% rename from sensor-iso/config/includes.chroot/usr/local/etc/zeek/login.zeek rename to hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/login.zeek diff --git a/sensor-iso/config/includes.chroot/usr/share/applications/configure-capture.desktop b/hedgehog-iso/config/includes.chroot/usr/share/applications/configure-capture.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/usr/share/applications/configure-capture.desktop rename to hedgehog-iso/config/includes.chroot/usr/share/applications/configure-capture.desktop diff --git a/sensor-iso/config/includes.chroot/usr/share/applications/configure-interfaces.desktop b/hedgehog-iso/config/includes.chroot/usr/share/applications/configure-interfaces.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/usr/share/applications/configure-interfaces.desktop rename to hedgehog-iso/config/includes.chroot/usr/share/applications/configure-interfaces.desktop diff --git a/sensor-iso/config/includes.chroot/usr/share/applications/hedgehog-kiosk.desktop b/hedgehog-iso/config/includes.chroot/usr/share/applications/hedgehog-kiosk.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/usr/share/applications/hedgehog-kiosk.desktop rename to hedgehog-iso/config/includes.chroot/usr/share/applications/hedgehog-kiosk.desktop diff --git a/sensor-iso/config/includes.chroot/usr/share/applications/hedgehog-readme.desktop b/hedgehog-iso/config/includes.chroot/usr/share/applications/hedgehog-readme.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/usr/share/applications/hedgehog-readme.desktop rename to hedgehog-iso/config/includes.chroot/usr/share/applications/hedgehog-readme.desktop diff --git a/sensor-iso/config/includes.chroot/usr/share/applications/sensor-services-full-restart.desktop b/hedgehog-iso/config/includes.chroot/usr/share/applications/sensor-services-full-restart.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/usr/share/applications/sensor-services-full-restart.desktop rename to hedgehog-iso/config/includes.chroot/usr/share/applications/sensor-services-full-restart.desktop diff --git a/sensor-iso/config/includes.chroot/usr/share/applications/sensor-services-status.desktop b/hedgehog-iso/config/includes.chroot/usr/share/applications/sensor-services-status.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/usr/share/applications/sensor-services-status.desktop rename to hedgehog-iso/config/includes.chroot/usr/share/applications/sensor-services-status.desktop diff --git a/sensor-iso/config/package-lists/build.list.chroot b/hedgehog-iso/config/package-lists/build.list.chroot similarity index 100% rename from sensor-iso/config/package-lists/build.list.chroot rename to hedgehog-iso/config/package-lists/build.list.chroot diff --git a/sensor-iso/config/package-lists/desktopmanager.list.chroot b/hedgehog-iso/config/package-lists/desktopmanager.list.chroot similarity index 100% rename from sensor-iso/config/package-lists/desktopmanager.list.chroot rename to hedgehog-iso/config/package-lists/desktopmanager.list.chroot diff --git a/sensor-iso/config/package-lists/grub.list.binary b/hedgehog-iso/config/package-lists/grub.list.binary similarity index 100% rename from sensor-iso/config/package-lists/grub.list.binary rename to hedgehog-iso/config/package-lists/grub.list.binary diff --git a/sensor-iso/config/package-lists/live.list.chroot b/hedgehog-iso/config/package-lists/live.list.chroot similarity index 100% rename from sensor-iso/config/package-lists/live.list.chroot rename to hedgehog-iso/config/package-lists/live.list.chroot diff --git a/sensor-iso/config/package-lists/net.list.chroot b/hedgehog-iso/config/package-lists/net.list.chroot similarity index 100% rename from sensor-iso/config/package-lists/net.list.chroot rename to hedgehog-iso/config/package-lists/net.list.chroot diff --git a/sensor-iso/config/package-lists/python.list.chroot b/hedgehog-iso/config/package-lists/python.list.chroot similarity index 100% rename from sensor-iso/config/package-lists/python.list.chroot rename to hedgehog-iso/config/package-lists/python.list.chroot diff --git a/sensor-iso/config/package-lists/sensor.list.chroot b/hedgehog-iso/config/package-lists/sensor.list.chroot similarity index 100% rename from sensor-iso/config/package-lists/sensor.list.chroot rename to hedgehog-iso/config/package-lists/sensor.list.chroot diff --git a/sensor-iso/config/package-lists/system.list.chroot b/hedgehog-iso/config/package-lists/system.list.chroot similarity index 100% rename from sensor-iso/config/package-lists/system.list.chroot rename to hedgehog-iso/config/package-lists/system.list.chroot diff --git a/sensor-iso/config/package-lists/virtualguest.list.chroot b/hedgehog-iso/config/package-lists/virtualguest.list.chroot similarity index 100% rename from sensor-iso/config/package-lists/virtualguest.list.chroot rename to hedgehog-iso/config/package-lists/virtualguest.list.chroot diff --git a/sensor-iso/interface/.gitignore b/hedgehog-iso/interface/.gitignore similarity index 100% rename from sensor-iso/interface/.gitignore rename to hedgehog-iso/interface/.gitignore diff --git a/sensor-iso/interface/init.sh b/hedgehog-iso/interface/init.sh similarity index 100% rename from sensor-iso/interface/init.sh rename to hedgehog-iso/interface/init.sh diff --git a/sensor-iso/interface/kiosk.service b/hedgehog-iso/interface/kiosk.service similarity index 100% rename from sensor-iso/interface/kiosk.service rename to hedgehog-iso/interface/kiosk.service diff --git a/sensor-iso/interface/requirements.txt b/hedgehog-iso/interface/requirements.txt similarity index 100% rename from sensor-iso/interface/requirements.txt rename to hedgehog-iso/interface/requirements.txt diff --git a/sensor-iso/interface/sensor_ctl/arkime/config.ini b/hedgehog-iso/interface/sensor_ctl/arkime/config.ini similarity index 100% rename from sensor-iso/interface/sensor_ctl/arkime/config.ini rename to hedgehog-iso/interface/sensor_ctl/arkime/config.ini diff --git a/sensor-iso/interface/sensor_ctl/arkime/rules.yml b/hedgehog-iso/interface/sensor_ctl/arkime/rules.yml similarity index 100% rename from sensor-iso/interface/sensor_ctl/arkime/rules.yml rename to hedgehog-iso/interface/sensor_ctl/arkime/rules.yml diff --git a/sensor-iso/interface/sensor_ctl/clean b/hedgehog-iso/interface/sensor_ctl/clean similarity index 100% rename from sensor-iso/interface/sensor_ctl/clean rename to hedgehog-iso/interface/sensor_ctl/clean diff --git a/sensor-iso/interface/sensor_ctl/clean.sh b/hedgehog-iso/interface/sensor_ctl/clean.sh similarity index 100% rename from sensor-iso/interface/sensor_ctl/clean.sh rename to hedgehog-iso/interface/sensor_ctl/clean.sh diff --git a/sensor-iso/interface/sensor_ctl/control.sh b/hedgehog-iso/interface/sensor_ctl/control.sh similarity index 100% rename from sensor-iso/interface/sensor_ctl/control.sh rename to hedgehog-iso/interface/sensor_ctl/control.sh diff --git a/sensor-iso/interface/sensor_ctl/control_vars.conf b/hedgehog-iso/interface/sensor_ctl/control_vars.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/control_vars.conf rename to hedgehog-iso/interface/sensor_ctl/control_vars.conf diff --git a/sensor-iso/interface/sensor_ctl/filebeat/filebeat.yml b/hedgehog-iso/interface/sensor_ctl/filebeat/filebeat.yml similarity index 100% rename from sensor-iso/interface/sensor_ctl/filebeat/filebeat.yml rename to hedgehog-iso/interface/sensor_ctl/filebeat/filebeat.yml diff --git a/sensor-iso/interface/sensor_ctl/filebeat/sensor_filebeat_local.sh b/hedgehog-iso/interface/sensor_ctl/filebeat/sensor_filebeat_local.sh similarity index 100% rename from sensor-iso/interface/sensor_ctl/filebeat/sensor_filebeat_local.sh rename to hedgehog-iso/interface/sensor_ctl/filebeat/sensor_filebeat_local.sh diff --git a/sensor-iso/interface/sensor_ctl/logstash-client-certificates/.gitignore b/hedgehog-iso/interface/sensor_ctl/logstash-client-certificates/.gitignore similarity index 100% rename from sensor-iso/interface/sensor_ctl/logstash-client-certificates/.gitignore rename to hedgehog-iso/interface/sensor_ctl/logstash-client-certificates/.gitignore diff --git a/sensor-iso/interface/sensor_ctl/miscbeat/filebeat.yml b/hedgehog-iso/interface/sensor_ctl/miscbeat/filebeat.yml similarity index 100% rename from sensor-iso/interface/sensor_ctl/miscbeat/filebeat.yml rename to hedgehog-iso/interface/sensor_ctl/miscbeat/filebeat.yml diff --git a/sensor-iso/interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh b/hedgehog-iso/interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh similarity index 100% rename from sensor-iso/interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh rename to hedgehog-iso/interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh diff --git a/sensor-iso/interface/sensor_ctl/reload b/hedgehog-iso/interface/sensor_ctl/reload similarity index 100% rename from sensor-iso/interface/sensor_ctl/reload rename to hedgehog-iso/interface/sensor_ctl/reload diff --git a/sensor-iso/interface/sensor_ctl/restart b/hedgehog-iso/interface/sensor_ctl/restart similarity index 100% rename from sensor-iso/interface/sensor_ctl/restart rename to hedgehog-iso/interface/sensor_ctl/restart diff --git a/sensor-iso/interface/sensor_ctl/scripts/log_disk_space.sh b/hedgehog-iso/interface/sensor_ctl/scripts/log_disk_space.sh similarity index 100% rename from sensor-iso/interface/sensor_ctl/scripts/log_disk_space.sh rename to hedgehog-iso/interface/sensor_ctl/scripts/log_disk_space.sh diff --git a/sensor-iso/interface/sensor_ctl/shutdown b/hedgehog-iso/interface/sensor_ctl/shutdown similarity index 100% rename from sensor-iso/interface/sensor_ctl/shutdown rename to hedgehog-iso/interface/sensor_ctl/shutdown diff --git a/sensor-iso/interface/sensor_ctl/start b/hedgehog-iso/interface/sensor_ctl/start similarity index 100% rename from sensor-iso/interface/sensor_ctl/start rename to hedgehog-iso/interface/sensor_ctl/start diff --git a/sensor-iso/interface/sensor_ctl/status b/hedgehog-iso/interface/sensor_ctl/status similarity index 100% rename from sensor-iso/interface/sensor_ctl/status rename to hedgehog-iso/interface/sensor_ctl/status diff --git a/sensor-iso/interface/sensor_ctl/stop b/hedgehog-iso/interface/sensor_ctl/stop similarity index 100% rename from sensor-iso/interface/sensor_ctl/stop rename to hedgehog-iso/interface/sensor_ctl/stop diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/arkime.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/arkime.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/arkime.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/arkime.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/beats.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/beats.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/beats.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/beats.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/clamav.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/clamav.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/clamav.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/clamav.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/documentation.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/documentation.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/documentation.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/documentation.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-aide.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-aide.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-aide.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-aide.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-auditlog.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-auditlog.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-auditlog.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-auditlog.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-kmsg.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-kmsg.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-kmsg.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-kmsg.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-metrics.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-metrics.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-metrics.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-metrics.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-systemd.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-systemd.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-systemd.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-systemd.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-thermal.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-thermal.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-thermal.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-thermal.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/netsniff.template b/hedgehog-iso/interface/sensor_ctl/supervisor.d/netsniff.template similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/netsniff.template rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/netsniff.template diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/prune.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/prune.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/prune.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/prune.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/supercronic.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/supercronic.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/supercronic.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/supercronic.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/suricata.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/suricata.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/suricata.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/suricata.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/tcpdump.template b/hedgehog-iso/interface/sensor_ctl/supervisor.d/tcpdump.template similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/tcpdump.template rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/tcpdump.template diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/zeek.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/zeek.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/zeek.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/zeek.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh b/hedgehog-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh rename to hedgehog-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh diff --git a/sensor-iso/interface/sensor_ctl/supervisor.init/supercronic_populate.sh b/hedgehog-iso/interface/sensor_ctl/supervisor.init/supercronic_populate.sh similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.init/supercronic_populate.sh rename to hedgehog-iso/interface/sensor_ctl/supervisor.init/supercronic_populate.sh diff --git a/sensor-iso/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh b/hedgehog-iso/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh rename to hedgehog-iso/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh diff --git a/sensor-iso/interface/sensor_ctl/supervisor.sh b/hedgehog-iso/interface/sensor_ctl/supervisor.sh similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.sh rename to hedgehog-iso/interface/sensor_ctl/supervisor.sh diff --git a/sensor-iso/interface/sensor_ctl/supervisord.conf b/hedgehog-iso/interface/sensor_ctl/supervisord.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisord.conf rename to hedgehog-iso/interface/sensor_ctl/supervisord.conf diff --git a/sensor-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek b/hedgehog-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek similarity index 100% rename from sensor-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek rename to hedgehog-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek diff --git a/sensor-iso/interface/sensor_ctl/zeek/intel/.gitignore b/hedgehog-iso/interface/sensor_ctl/zeek/intel/.gitignore similarity index 100% rename from sensor-iso/interface/sensor_ctl/zeek/intel/.gitignore rename to hedgehog-iso/interface/sensor_ctl/zeek/intel/.gitignore diff --git a/sensor-iso/interface/sensor_interface/__init__.py b/hedgehog-iso/interface/sensor_interface/__init__.py similarity index 100% rename from sensor-iso/interface/sensor_interface/__init__.py rename to hedgehog-iso/interface/sensor_interface/__init__.py diff --git a/sensor-iso/interface/sensor_interface/__main__.py b/hedgehog-iso/interface/sensor_interface/__main__.py similarity index 100% rename from sensor-iso/interface/sensor_interface/__main__.py rename to hedgehog-iso/interface/sensor_interface/__main__.py diff --git a/sensor-iso/interface/sensor_interface/routes.py b/hedgehog-iso/interface/sensor_interface/routes.py similarity index 100% rename from sensor-iso/interface/sensor_interface/routes.py rename to hedgehog-iso/interface/sensor_interface/routes.py diff --git a/sensor-iso/interface/sensor_interface/static/css/custom.css b/hedgehog-iso/interface/sensor_interface/static/css/custom.css similarity index 100% rename from sensor-iso/interface/sensor_interface/static/css/custom.css rename to hedgehog-iso/interface/sensor_interface/static/css/custom.css diff --git a/sensor-iso/interface/sensor_interface/static/css/freeboard.min.css b/hedgehog-iso/interface/sensor_interface/static/css/freeboard.min.css similarity index 100% rename from sensor-iso/interface/sensor_interface/static/css/freeboard.min.css rename to hedgehog-iso/interface/sensor_interface/static/css/freeboard.min.css diff --git a/sensor-iso/interface/sensor_interface/static/css/material.css b/hedgehog-iso/interface/sensor_interface/static/css/material.css similarity index 100% rename from sensor-iso/interface/sensor_interface/static/css/material.css rename to hedgehog-iso/interface/sensor_interface/static/css/material.css diff --git a/sensor-iso/interface/sensor_interface/static/dashboard/dashboard.json b/hedgehog-iso/interface/sensor_interface/static/dashboard/dashboard.json similarity index 100% rename from sensor-iso/interface/sensor_interface/static/dashboard/dashboard.json rename to hedgehog-iso/interface/sensor_interface/static/dashboard/dashboard.json diff --git a/sensor-iso/interface/sensor_interface/static/icons/icons.woff2 b/hedgehog-iso/interface/sensor_interface/static/icons/icons.woff2 similarity index 100% rename from sensor-iso/interface/sensor_interface/static/icons/icons.woff2 rename to hedgehog-iso/interface/sensor_interface/static/icons/icons.woff2 diff --git a/sensor-iso/interface/sensor_interface/static/img/dropdown-arrow.png b/hedgehog-iso/interface/sensor_interface/static/img/dropdown-arrow.png similarity index 100% rename from sensor-iso/interface/sensor_interface/static/img/dropdown-arrow.png rename to hedgehog-iso/interface/sensor_interface/static/img/dropdown-arrow.png diff --git a/sensor-iso/interface/sensor_interface/static/img/glyphicons-blackboard.png b/hedgehog-iso/interface/sensor_interface/static/img/glyphicons-blackboard.png similarity index 100% rename from sensor-iso/interface/sensor_interface/static/img/glyphicons-blackboard.png rename to hedgehog-iso/interface/sensor_interface/static/img/glyphicons-blackboard.png diff --git a/sensor-iso/interface/sensor_interface/static/img/glyphicons-halflings-white.png b/hedgehog-iso/interface/sensor_interface/static/img/glyphicons-halflings-white.png similarity index 100% rename from sensor-iso/interface/sensor_interface/static/img/glyphicons-halflings-white.png rename to hedgehog-iso/interface/sensor_interface/static/img/glyphicons-halflings-white.png diff --git a/sensor-iso/interface/sensor_interface/static/img/glyphicons-halflings.png b/hedgehog-iso/interface/sensor_interface/static/img/glyphicons-halflings.png similarity index 100% rename from sensor-iso/interface/sensor_interface/static/img/glyphicons-halflings.png rename to hedgehog-iso/interface/sensor_interface/static/img/glyphicons-halflings.png diff --git a/sensor-iso/interface/sensor_interface/static/img/glyphicons-log-in.png b/hedgehog-iso/interface/sensor_interface/static/img/glyphicons-log-in.png similarity index 100% rename from sensor-iso/interface/sensor_interface/static/img/glyphicons-log-in.png rename to hedgehog-iso/interface/sensor_interface/static/img/glyphicons-log-in.png diff --git a/sensor-iso/interface/sensor_interface/static/img/glyphicons-log-out.png b/hedgehog-iso/interface/sensor_interface/static/img/glyphicons-log-out.png similarity index 100% rename from sensor-iso/interface/sensor_interface/static/img/glyphicons-log-out.png rename to hedgehog-iso/interface/sensor_interface/static/img/glyphicons-log-out.png diff --git a/sensor-iso/interface/sensor_interface/static/js/custom.js b/hedgehog-iso/interface/sensor_interface/static/js/custom.js similarity index 100% rename from sensor-iso/interface/sensor_interface/static/js/custom.js rename to hedgehog-iso/interface/sensor_interface/static/js/custom.js diff --git a/sensor-iso/interface/sensor_interface/static/js/freeboard.min.js b/hedgehog-iso/interface/sensor_interface/static/js/freeboard.min.js similarity index 100% rename from sensor-iso/interface/sensor_interface/static/js/freeboard.min.js rename to hedgehog-iso/interface/sensor_interface/static/js/freeboard.min.js diff --git a/sensor-iso/interface/sensor_interface/static/js/freeboard.plugins.min.js b/hedgehog-iso/interface/sensor_interface/static/js/freeboard.plugins.min.js similarity index 100% rename from sensor-iso/interface/sensor_interface/static/js/freeboard.plugins.min.js rename to hedgehog-iso/interface/sensor_interface/static/js/freeboard.plugins.min.js diff --git a/sensor-iso/interface/sensor_interface/static/js/freeboard.thirdparty.min.js b/hedgehog-iso/interface/sensor_interface/static/js/freeboard.thirdparty.min.js similarity index 100% rename from sensor-iso/interface/sensor_interface/static/js/freeboard.thirdparty.min.js rename to hedgehog-iso/interface/sensor_interface/static/js/freeboard.thirdparty.min.js diff --git a/sensor-iso/interface/sensor_interface/static/js/freeboard_plugins.min.js b/hedgehog-iso/interface/sensor_interface/static/js/freeboard_plugins.min.js similarity index 100% rename from sensor-iso/interface/sensor_interface/static/js/freeboard_plugins.min.js rename to hedgehog-iso/interface/sensor_interface/static/js/freeboard_plugins.min.js diff --git a/sensor-iso/interface/sensor_interface/static/js/jquery.sparkline.min.js b/hedgehog-iso/interface/sensor_interface/static/js/jquery.sparkline.min.js similarity index 100% rename from sensor-iso/interface/sensor_interface/static/js/jquery.sparkline.min.js rename to hedgehog-iso/interface/sensor_interface/static/js/jquery.sparkline.min.js diff --git a/sensor-iso/interface/sensor_interface/static/js/material.min.js b/hedgehog-iso/interface/sensor_interface/static/js/material.min.js similarity index 100% rename from sensor-iso/interface/sensor_interface/static/js/material.min.js rename to hedgehog-iso/interface/sensor_interface/static/js/material.min.js diff --git a/sensor-iso/interface/sensor_interface/sysquery/__init__.py b/hedgehog-iso/interface/sensor_interface/sysquery/__init__.py similarity index 100% rename from sensor-iso/interface/sensor_interface/sysquery/__init__.py rename to hedgehog-iso/interface/sensor_interface/sysquery/__init__.py diff --git a/sensor-iso/interface/sensor_interface/sysquery/sys_service.py b/hedgehog-iso/interface/sensor_interface/sysquery/sys_service.py similarity index 100% rename from sensor-iso/interface/sensor_interface/sysquery/sys_service.py rename to hedgehog-iso/interface/sensor_interface/sysquery/sys_service.py diff --git a/sensor-iso/interface/sensor_interface/templates/base.html b/hedgehog-iso/interface/sensor_interface/templates/base.html similarity index 100% rename from sensor-iso/interface/sensor_interface/templates/base.html rename to hedgehog-iso/interface/sensor_interface/templates/base.html diff --git a/sensor-iso/interface/sensor_interface/templates/buttons.html b/hedgehog-iso/interface/sensor_interface/templates/buttons.html similarity index 100% rename from sensor-iso/interface/sensor_interface/templates/buttons.html rename to hedgehog-iso/interface/sensor_interface/templates/buttons.html diff --git a/sensor-iso/interface/sensor_interface/templates/system_block.html b/hedgehog-iso/interface/sensor_interface/templates/system_block.html similarity index 100% rename from sensor-iso/interface/sensor_interface/templates/system_block.html rename to hedgehog-iso/interface/sensor_interface/templates/system_block.html diff --git a/sensor-iso/vagrant/Vagrantfile b/hedgehog-iso/vagrant/Vagrantfile similarity index 100% rename from sensor-iso/vagrant/Vagrantfile rename to hedgehog-iso/vagrant/Vagrantfile diff --git a/sensor-iso/vagrant/ssh_config b/hedgehog-iso/vagrant/ssh_config similarity index 100% rename from sensor-iso/vagrant/ssh_config rename to hedgehog-iso/vagrant/ssh_config diff --git a/sensor-iso/yara/Dockerfile b/hedgehog-iso/yara/Dockerfile similarity index 100% rename from sensor-iso/yara/Dockerfile rename to hedgehog-iso/yara/Dockerfile diff --git a/sensor-iso/yara/build-docker-image.sh b/hedgehog-iso/yara/build-docker-image.sh similarity index 100% rename from sensor-iso/yara/build-docker-image.sh rename to hedgehog-iso/yara/build-docker-image.sh diff --git a/sensor-iso/yara/build-yara-deb.sh b/hedgehog-iso/yara/build-yara-deb.sh similarity index 100% rename from sensor-iso/yara/build-yara-deb.sh rename to hedgehog-iso/yara/build-yara-deb.sh diff --git a/sensor-raspi/.gitignore b/hedgehog-raspi/.gitignore similarity index 100% rename from sensor-raspi/.gitignore rename to hedgehog-raspi/.gitignore diff --git a/sensor-raspi/Dockerfile b/hedgehog-raspi/Dockerfile similarity index 95% rename from sensor-raspi/Dockerfile rename to hedgehog-raspi/Dockerfile index 7338c21a3..dcdcb9468 100644 --- a/sensor-raspi/Dockerfile +++ b/hedgehog-raspi/Dockerfile @@ -5,7 +5,7 @@ FROM ghcr.io/mmguero/qemu-live-iso:latest LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' -LABEL org.opencontainers.image.documentation='https://github.com/idaholab/Malcolm/blob/main/sensor-iso/README.md' +LABEL org.opencontainers.image.documentation='https://github.com/idaholab/Malcolm/blob/main/hedgehog-iso/README.md' LABEL org.opencontainers.image.source='https://github.com/idaholab/Malcolm' LABEL org.opencontainers.image.vendor='Idaho National Laboratory' LABEL org.opencontainers.image.title='ghcr.io/idaholab/malcolm/hedgehog-raspi' diff --git a/sensor-raspi/Makefile b/hedgehog-raspi/Makefile similarity index 100% rename from sensor-raspi/Makefile rename to hedgehog-raspi/Makefile diff --git a/sensor-raspi/README.md b/hedgehog-raspi/README.md similarity index 99% rename from sensor-raspi/README.md rename to hedgehog-raspi/README.md index 9b2bc1c42..fe63ddbee 100644 --- a/sensor-raspi/README.md +++ b/hedgehog-raspi/README.md @@ -11,7 +11,7 @@ clone this repository: ```shell git clone https://github.com/idaholab/Malcolm.git -cd Malcolm/sensor-raspi +cd Malcolm/hedgehog-raspi ``` For this you will first need to install the following packages on a diff --git a/sensor-raspi/attribution.txt b/hedgehog-raspi/attribution.txt similarity index 100% rename from sensor-raspi/attribution.txt rename to hedgehog-raspi/attribution.txt diff --git a/sensor-raspi/build_via_vagrant.sh b/hedgehog-raspi/build_via_vagrant.sh similarity index 96% rename from sensor-raspi/build_via_vagrant.sh rename to hedgehog-raspi/build_via_vagrant.sh index 6141a6aa3..396f51d88 100755 --- a/sensor-raspi/build_via_vagrant.sh +++ b/hedgehog-raspi/build_via_vagrant.sh @@ -102,7 +102,7 @@ YML_IMAGE_VERSION="$(grep -P "^\s+image:.*/malcolm/" "$SCRIPT_PATH"/../docker-co echo "VCS_REVSION=$( git rev-parse --short HEAD 2>/dev/null || echo main )" >> "$SCRIPT_PATH"/shared/environment.chroot trap cleanup_shared_and_docs EXIT -vm_execute "sudo bash -c \"whoami && cd /Malcolm/sensor-raspi && pwd && make ${IMAGE}${XZ_EXT}\"" +vm_execute "sudo bash -c \"whoami && cd /Malcolm/hedgehog-raspi && pwd && make ${IMAGE}${XZ_EXT}\"" if [[ -n $NEED_SHUTDOWN ]]; then echo "Shutting down $VM_NAME..." >&2 diff --git a/sensor-raspi/debian/salsa-ci.yml b/hedgehog-raspi/debian/salsa-ci.yml similarity index 100% rename from sensor-raspi/debian/salsa-ci.yml rename to hedgehog-raspi/debian/salsa-ci.yml diff --git a/sensor-raspi/debian/yamllint.yml b/hedgehog-raspi/debian/yamllint.yml similarity index 100% rename from sensor-raspi/debian/yamllint.yml rename to hedgehog-raspi/debian/yamllint.yml diff --git a/sensor-raspi/generate-recipe.py b/hedgehog-raspi/generate-recipe.py similarity index 99% rename from sensor-raspi/generate-recipe.py rename to hedgehog-raspi/generate-recipe.py index e40b8a638..4ad1535c7 100755 --- a/sensor-raspi/generate-recipe.py +++ b/hedgehog-raspi/generate-recipe.py @@ -7,7 +7,7 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) MALCOLM_DIR = os.path.dirname(SCRIPT_DIR) -SENSOR_DIR = os.path.join(MALCOLM_DIR, 'sensor-iso') +SENSOR_DIR = os.path.join(MALCOLM_DIR, 'hedgehog-iso') # pylint: disable=invalid-name diff --git a/sensor-raspi/raspi_master.yaml b/hedgehog-raspi/raspi_master.yaml similarity index 100% rename from sensor-raspi/raspi_master.yaml rename to hedgehog-raspi/raspi_master.yaml diff --git a/sensor-raspi/rootfs/boot/firmware/sysconf.txt b/hedgehog-raspi/rootfs/boot/firmware/sysconf.txt similarity index 100% rename from sensor-raspi/rootfs/boot/firmware/sysconf.txt rename to hedgehog-raspi/rootfs/boot/firmware/sysconf.txt diff --git a/sensor-raspi/rootfs/etc/fstab b/hedgehog-raspi/rootfs/etc/fstab similarity index 100% rename from sensor-raspi/rootfs/etc/fstab rename to hedgehog-raspi/rootfs/etc/fstab diff --git a/sensor-raspi/rootfs/etc/initramfs-tools/hooks/rpi-resizerootfs b/hedgehog-raspi/rootfs/etc/initramfs-tools/hooks/rpi-resizerootfs similarity index 100% rename from sensor-raspi/rootfs/etc/initramfs-tools/hooks/rpi-resizerootfs rename to hedgehog-raspi/rootfs/etc/initramfs-tools/hooks/rpi-resizerootfs diff --git a/sensor-raspi/rootfs/etc/initramfs-tools/scripts/local-bottom/rpi-resizerootfs b/hedgehog-raspi/rootfs/etc/initramfs-tools/scripts/local-bottom/rpi-resizerootfs similarity index 100% rename from sensor-raspi/rootfs/etc/initramfs-tools/scripts/local-bottom/rpi-resizerootfs rename to hedgehog-raspi/rootfs/etc/initramfs-tools/scripts/local-bottom/rpi-resizerootfs diff --git a/sensor-raspi/rootfs/etc/network/interfaces.d/eth0 b/hedgehog-raspi/rootfs/etc/network/interfaces.d/eth0 similarity index 100% rename from sensor-raspi/rootfs/etc/network/interfaces.d/eth0 rename to hedgehog-raspi/rootfs/etc/network/interfaces.d/eth0 diff --git a/sensor-raspi/rootfs/etc/network/interfaces.d/wlan0 b/hedgehog-raspi/rootfs/etc/network/interfaces.d/wlan0 similarity index 100% rename from sensor-raspi/rootfs/etc/network/interfaces.d/wlan0 rename to hedgehog-raspi/rootfs/etc/network/interfaces.d/wlan0 diff --git a/sensor-raspi/rootfs/etc/systemd/system/rpi-generate-ssh-host-keys.service b/hedgehog-raspi/rootfs/etc/systemd/system/rpi-generate-ssh-host-keys.service similarity index 100% rename from sensor-raspi/rootfs/etc/systemd/system/rpi-generate-ssh-host-keys.service rename to hedgehog-raspi/rootfs/etc/systemd/system/rpi-generate-ssh-host-keys.service diff --git a/sensor-raspi/rootfs/etc/systemd/system/rpi-reconfigure-raspi-firmware.service b/hedgehog-raspi/rootfs/etc/systemd/system/rpi-reconfigure-raspi-firmware.service similarity index 100% rename from sensor-raspi/rootfs/etc/systemd/system/rpi-reconfigure-raspi-firmware.service rename to hedgehog-raspi/rootfs/etc/systemd/system/rpi-reconfigure-raspi-firmware.service diff --git a/sensor-raspi/rootfs/etc/systemd/system/rpi-set-sysconf.service b/hedgehog-raspi/rootfs/etc/systemd/system/rpi-set-sysconf.service similarity index 100% rename from sensor-raspi/rootfs/etc/systemd/system/rpi-set-sysconf.service rename to hedgehog-raspi/rootfs/etc/systemd/system/rpi-set-sysconf.service diff --git a/sensor-raspi/rootfs/etc/wpa_supplicant/wpa_supplicant.conf b/hedgehog-raspi/rootfs/etc/wpa_supplicant/wpa_supplicant.conf similarity index 100% rename from sensor-raspi/rootfs/etc/wpa_supplicant/wpa_supplicant.conf rename to hedgehog-raspi/rootfs/etc/wpa_supplicant/wpa_supplicant.conf diff --git a/sensor-raspi/rootfs/usr/local/sbin/rpi-set-sysconf b/hedgehog-raspi/rootfs/usr/local/sbin/rpi-set-sysconf similarity index 100% rename from sensor-raspi/rootfs/usr/local/sbin/rpi-set-sysconf rename to hedgehog-raspi/rootfs/usr/local/sbin/rpi-set-sysconf diff --git a/sensor-raspi/sensor_install.sh b/hedgehog-raspi/sensor_install.sh similarity index 99% rename from sensor-raspi/sensor_install.sh rename to hedgehog-raspi/sensor_install.sh index d060e2ba7..799a326de 100644 --- a/sensor-raspi/sensor_install.sh +++ b/hedgehog-raspi/sensor_install.sh @@ -329,7 +329,7 @@ install_deps() { deps+=$(tr '\n' ' ' < "$file") done - # Remove Sensor-ISO packages not relevant to RPI + # Remove hedgehog-iso packages not relevant to RPI # Rar is excluded because Debian doesn't have an ARM package # htpdate removed because repo version doesn't support https # aide is removed as we're not applying the same hardening requirements ot the rpi image diff --git a/sensor-raspi/vagrant/Vagrantfile b/hedgehog-raspi/vagrant/Vagrantfile similarity index 100% rename from sensor-raspi/vagrant/Vagrantfile rename to hedgehog-raspi/vagrant/Vagrantfile diff --git a/sensor-raspi/vagrant/ssh_config b/hedgehog-raspi/vagrant/ssh_config similarity index 100% rename from sensor-raspi/vagrant/ssh_config rename to hedgehog-raspi/vagrant/ssh_config From 58f74b2a4ed79dda4ca3a75b44d7c5dce60fb1e8 Mon Sep 17 00:00:00 2001 From: SG Date: Fri, 16 Feb 2024 12:08:53 -0700 Subject: [PATCH 04/79] fix the way we do environment variables in local.zeek (close idaholab/Malcolm#413) --- config/zeek-live.env.example | 2 +- config/zeek.env.example | 2 +- docs/malcolm-config.md | 4 +- .../usr/local/etc/zeek/local.zeek | 62 ++++++++++--------- zeek/config/local.zeek | 62 ++++++++++--------- 5 files changed, 68 insertions(+), 64 deletions(-) diff --git a/config/zeek-live.env.example b/config/zeek-live.env.example index 944833e85..abf8c7a4d 100644 --- a/config/zeek-live.env.example +++ b/config/zeek-live.env.example @@ -1,7 +1,7 @@ # Whether or not Zeek should monitor live traffic on a local # interface (PCAP_IFACE variable below specifies capture interfaces) ZEEK_LIVE_CAPTURE=false -# Set ZEEK_DISABLE_STATS to blank to generate stats.log and capture_loss.log +# Set ZEEK_DISABLE_STATS to false or blank to generate stats.log and capture_loss.log ZEEK_DISABLE_STATS=true ZEEK_PCAP_PROCESSOR=false diff --git a/config/zeek.env.example b/config/zeek.env.example index 966a1bfa2..06c72e8de 100644 --- a/config/zeek.env.example +++ b/config/zeek.env.example @@ -56,7 +56,7 @@ EXTRACTED_FILE_HTTP_SERVER_MAGIC=false # HTTP server will look in subdirectories for requested filename (e.g., in "/quarantined" and "/preserved") EXTRACTED_FILE_HTTP_SERVER_RECURSIVE=true # Environment variables for tweaking Zeek at runtime (see local.zeek) -# Set to any non-blank value to disable the corresponding feature +# Set to true to disable the corresponding feature ZEEK_DISABLE_HASH_ALL_FILES= ZEEK_DISABLE_LOG_PASSWORDS= ZEEK_DISABLE_SSL_VALIDATE_CERTS= diff --git a/docs/malcolm-config.md b/docs/malcolm-config.md index 7caac0274..5f6bf38b8 100644 --- a/docs/malcolm-config.md +++ b/docs/malcolm-config.md @@ -111,9 +111,9 @@ Although the configuration script automates many of the following configuration - `VTOT_API2_KEY` – used to specify a [VirusTotal Public API v.20](https://www.virustotal.com/en/documentation/public-api/) key, which, if specified, will be used to submit hashes of [Zeek-extracted files](file-scanning.md#ZeekFileExtraction) to VirusTotal - `ZEEK_AUTO_ANALYZE_PCAP_FILES` – if set to `true`, all PCAP files imported into Malcolm will automatically be analyzed by Zeek, and the resulting logs will also be imported (default `false`) - `ZEEK_AUTO_ANALYZE_PCAP_THREADS` – the number of threads available to Malcolm for analyzing Zeek logs (default `1`) - - `ZEEK_DISABLE_…` - if set to any non-blank value, each of these variables can be used to disable a certain Zeek function when it analyzes PCAP files (for example, setting `ZEEK_DISABLE_LOG_PASSWORDS` to `true` to disable logging of cleartext passwords) + - `ZEEK_DISABLE_…` - if set to `true`, each of these variables can be used to disable a certain Zeek function when it analyzes PCAP files (for example, setting `ZEEK_DISABLE_LOG_PASSWORDS` to `true` to disable logging of cleartext passwords) - `ZEEK_…_PORTS` - used to specify non-default ports to register certain Zeek analyzers (e.g., `ZEEK_SYNCHROPHASOR_PORTS` for the [ICSNPP-Synchrophasor analyzer](https://github.com/cisagov/icsnpp-synchrophasor/), `ZEEK_GENISYS_PORTS` for the [ICSNPP-Genisys analyzer](https://github.com/cisagov/icsnpp-genisys/), and `ZEEK_ENIP_PORTS` for the [ICSNPP-Ethernet/IP analyzer](https://github.com/cisagov/icsnpp-enip/)) formatted as a comma-separated list of [Zeek ports](https://docs.zeek.org/en/master/scripting/basics.html#port) (e.g., `12345/tcp` or `4041/tcp,4042/udp`) - - `ZEEK_DISABLE_ICS_ALL` and `ZEEK_DISABLE_ICS_…` - if set to any non-blank value, these variables can be used to disable Zeek's protocol analyzers for Operational Technology/Industrial Control Systems (OT/ICS) protocols + - `ZEEK_DISABLE_ICS_ALL` and `ZEEK_DISABLE_ICS_…` - if set to `true`, these variables can be used to disable Zeek's protocol analyzers for Operational Technology/Industrial Control Systems (OT/ICS) protocols - `ZEEK_DISABLE_BEST_GUESS_ICS` - see ["Best Guess" Fingerprinting for ICS Protocols](ics-best-guess.md#ICSBestGuess) - `ZEEK_EXTRACTOR_MODE` – determines the file extraction behavior for file transfers detected by Zeek; see [Automatic file extraction and scanning](file-scanning.md#ZeekFileExtraction) for more details - `ZEEK_INTEL_FEED_SINCE` - when querying a [TAXII](zeek-intel.md#ZeekIntelSTIX) or [MISP](zeek-intel.md#ZeekIntelMISP) feed, only process threat indicators created or modified since the time represented by this value; it may be either a fixed date/time (`01/01/2021`) or relative interval (`30 days ago`) diff --git a/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek index 7d3b0ccee..717c4cd82 100644 --- a/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek +++ b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek @@ -4,42 +4,44 @@ ##! https://docs.zeek.org/en/stable/script-reference/scripts.html ##! https://github.com/zeek/zeek/blob/master/scripts/site/local.zeek -global disable_stats = (getenv("ZEEK_DISABLE_STATS") == "") ? F : T; -global disable_hash_all_files = (getenv("ZEEK_DISABLE_HASH_ALL_FILES") == "") ? F : T; -global disable_log_passwords = (getenv("ZEEK_DISABLE_LOG_PASSWORDS") == "") ? F : T; -global disable_ssl_validate_certs = (getenv("ZEEK_DISABLE_SSL_VALIDATE_CERTS") == "") ? F : T; -global disable_track_all_assets = (getenv("ZEEK_DISABLE_TRACK_ALL_ASSETS") == "") ? F : T; -global disable_best_guess_ics = (getenv("ZEEK_DISABLE_BEST_GUESS_ICS") == "") ? F : T; -global synchrophasor_detailed = (getenv("ZEEK_SYNCHROPHASOR_DETAILED") == "") ? F : T; +global true_regex: pattern = /^\s*(?i:t(rue)?|y(es)?|on|1)\s*$/; + +global disable_stats = (getenv("ZEEK_DISABLE_STATS") == true_regex) ? T : F; +global disable_hash_all_files = (getenv("ZEEK_DISABLE_HASH_ALL_FILES") == true_regex) ? T : F; +global disable_log_passwords = (getenv("ZEEK_DISABLE_LOG_PASSWORDS") == true_regex) ? T : F; +global disable_ssl_validate_certs = (getenv("ZEEK_DISABLE_SSL_VALIDATE_CERTS") == true_regex) ? T : F; +global disable_track_all_assets = (getenv("ZEEK_DISABLE_TRACK_ALL_ASSETS") == true_regex) ? T : F; +global disable_best_guess_ics = (getenv("ZEEK_DISABLE_BEST_GUESS_ICS") == true_regex) ? T : F; +global synchrophasor_detailed = (getenv("ZEEK_SYNCHROPHASOR_DETAILED") == true_regex) ? T : F; global synchrophasor_ports_str = getenv("ZEEK_SYNCHROPHASOR_PORTS"); global genisys_ports_str = getenv("ZEEK_GENISYS_PORTS"); global enip_ports_str = getenv("ZEEK_ENIP_PORTS"); global zeek_local_nets_str = getenv("ZEEK_LOCAL_NETS"); -global disable_spicy_dhcp = (getenv("ZEEK_DISABLE_SPICY_DHCP") == "") ? F : T; -global disable_spicy_dns = (getenv("ZEEK_DISABLE_SPICY_DNS") == "") ? F : T; -global disable_spicy_http = (getenv("ZEEK_DISABLE_SPICY_HTTP") == "") ? F : T; -global disable_spicy_ipsec = (getenv("ZEEK_DISABLE_SPICY_IPSEC") == "") ? F : T; -global disable_spicy_ldap = (getenv("ZEEK_DISABLE_SPICY_LDAP") == "") ? F : T; -global disable_spicy_openvpn = (getenv("ZEEK_DISABLE_SPICY_OPENVPN") == "") ? F : T; -global disable_spicy_quic = (getenv("ZEEK_DISABLE_SPICY_QUIC") == "") ? F : T; -global disable_spicy_stun = (getenv("ZEEK_DISABLE_SPICY_STUN") == "") ? F : T; -global disable_spicy_tailscale = (getenv("ZEEK_DISABLE_SPICY_TAILSCALE") == "") ? F : T; -global disable_spicy_tftp = (getenv("ZEEK_DISABLE_SPICY_TFTP") == "") ? F : T; -global disable_spicy_wireguard = (getenv("ZEEK_DISABLE_SPICY_WIREGUARD") == "") ? F : T; +global disable_spicy_dhcp = (getenv("ZEEK_DISABLE_SPICY_DHCP") == true_regex) ? T : F; +global disable_spicy_dns = (getenv("ZEEK_DISABLE_SPICY_DNS") == true_regex) ? T : F; +global disable_spicy_http = (getenv("ZEEK_DISABLE_SPICY_HTTP") == true_regex) ? T : F; +global disable_spicy_ipsec = (getenv("ZEEK_DISABLE_SPICY_IPSEC") == true_regex) ? T : F; +global disable_spicy_ldap = (getenv("ZEEK_DISABLE_SPICY_LDAP") == true_regex) ? T : F; +global disable_spicy_openvpn = (getenv("ZEEK_DISABLE_SPICY_OPENVPN") == true_regex) ? T : F; +global disable_spicy_quic = (getenv("ZEEK_DISABLE_SPICY_QUIC") == true_regex) ? T : F; +global disable_spicy_stun = (getenv("ZEEK_DISABLE_SPICY_STUN") == true_regex) ? T : F; +global disable_spicy_tailscale = (getenv("ZEEK_DISABLE_SPICY_TAILSCALE") == true_regex) ? T : F; +global disable_spicy_tftp = (getenv("ZEEK_DISABLE_SPICY_TFTP") == true_regex) ? T : F; +global disable_spicy_wireguard = (getenv("ZEEK_DISABLE_SPICY_WIREGUARD") == true_regex) ? T : F; -global disable_ics_all = (getenv("ZEEK_DISABLE_ICS_ALL") == "") ? F : T; -global disable_ics_bacnet = (getenv("ZEEK_DISABLE_ICS_BACNET") == "") ? F : T; -global disable_ics_bsap = (getenv("ZEEK_DISABLE_ICS_BSAP") == "") ? F : T; -global disable_ics_dnp3 = (getenv("ZEEK_DISABLE_ICS_DNP3") == "") ? F : T; -global disable_ics_enip = (getenv("ZEEK_DISABLE_ICS_ENIP") == "") ? F : T; -global disable_ics_ethercat = (getenv("ZEEK_DISABLE_ICS_ETHERCAT") == "") ? F : T; -global disable_ics_genisys = (getenv("ZEEK_DISABLE_ICS_GENISYS") == "") ? F : T; -global disable_ics_opcua_binary = (getenv("ZEEK_DISABLE_ICS_OPCUA_BINARY") == "") ? F : T; -global disable_ics_modbus = (getenv("ZEEK_DISABLE_ICS_MODBUS") == "") ? F : T; -global disable_ics_profinet = (getenv("ZEEK_DISABLE_ICS_PROFINET") == "") ? F : T; -global disable_ics_s7comm = (getenv("ZEEK_DISABLE_ICS_S7COMM") == "") ? F : T; -global disable_ics_synchrophasor = (getenv("ZEEK_DISABLE_ICS_SYNCHROPHASOR") == "") ? F : T; +global disable_ics_all = (getenv("ZEEK_DISABLE_ICS_ALL") == true_regex) ? T : F; +global disable_ics_bacnet = (getenv("ZEEK_DISABLE_ICS_BACNET") == true_regex) ? T : F; +global disable_ics_bsap = (getenv("ZEEK_DISABLE_ICS_BSAP") == true_regex) ? T : F; +global disable_ics_dnp3 = (getenv("ZEEK_DISABLE_ICS_DNP3") == true_regex) ? T : F; +global disable_ics_enip = (getenv("ZEEK_DISABLE_ICS_ENIP") == true_regex) ? T : F; +global disable_ics_ethercat = (getenv("ZEEK_DISABLE_ICS_ETHERCAT") == true_regex) ? T : F; +global disable_ics_genisys = (getenv("ZEEK_DISABLE_ICS_GENISYS") == true_regex) ? T : F; +global disable_ics_opcua_binary = (getenv("ZEEK_DISABLE_ICS_OPCUA_BINARY") == true_regex) ? T : F; +global disable_ics_modbus = (getenv("ZEEK_DISABLE_ICS_MODBUS") == true_regex) ? T : F; +global disable_ics_profinet = (getenv("ZEEK_DISABLE_ICS_PROFINET") == true_regex) ? T : F; +global disable_ics_s7comm = (getenv("ZEEK_DISABLE_ICS_S7COMM") == true_regex) ? T : F; +global disable_ics_synchrophasor = (getenv("ZEEK_DISABLE_ICS_SYNCHROPHASOR") == true_regex) ? T : F; redef Broker::default_listen_address = "127.0.0.1"; redef ignore_checksums = T; diff --git a/zeek/config/local.zeek b/zeek/config/local.zeek index a610c6c5a..d1f89808e 100644 --- a/zeek/config/local.zeek +++ b/zeek/config/local.zeek @@ -4,42 +4,44 @@ ##! https://docs.zeek.org/en/stable/script-reference/scripts.html ##! https://github.com/zeek/zeek/blob/master/scripts/site/local.zeek -global disable_stats = (getenv("ZEEK_DISABLE_STATS") == "") ? F : T; -global disable_hash_all_files = (getenv("ZEEK_DISABLE_HASH_ALL_FILES") == "") ? F : T; -global disable_log_passwords = (getenv("ZEEK_DISABLE_LOG_PASSWORDS") == "") ? F : T; -global disable_ssl_validate_certs = (getenv("ZEEK_DISABLE_SSL_VALIDATE_CERTS") == "") ? F : T; -global disable_track_all_assets = (getenv("ZEEK_DISABLE_TRACK_ALL_ASSETS") == "") ? F : T; -global disable_best_guess_ics = (getenv("ZEEK_DISABLE_BEST_GUESS_ICS") == "") ? F : T; -global synchrophasor_detailed = (getenv("ZEEK_SYNCHROPHASOR_DETAILED") == "") ? F : T; +global true_regex: pattern = /^\s*(?i:t(rue)?|y(es)?|on|1)\s*$/; + +global disable_stats = (getenv("ZEEK_DISABLE_STATS") == true_regex) ? T : F; +global disable_hash_all_files = (getenv("ZEEK_DISABLE_HASH_ALL_FILES") == true_regex) ? T : F; +global disable_log_passwords = (getenv("ZEEK_DISABLE_LOG_PASSWORDS") == true_regex) ? T : F; +global disable_ssl_validate_certs = (getenv("ZEEK_DISABLE_SSL_VALIDATE_CERTS") == true_regex) ? T : F; +global disable_track_all_assets = (getenv("ZEEK_DISABLE_TRACK_ALL_ASSETS") == true_regex) ? T : F; +global disable_best_guess_ics = (getenv("ZEEK_DISABLE_BEST_GUESS_ICS") == true_regex) ? T : F; +global synchrophasor_detailed = (getenv("ZEEK_SYNCHROPHASOR_DETAILED") == true_regex) ? T : F; global synchrophasor_ports_str = getenv("ZEEK_SYNCHROPHASOR_PORTS"); global genisys_ports_str = getenv("ZEEK_GENISYS_PORTS"); global enip_ports_str = getenv("ZEEK_ENIP_PORTS"); global zeek_local_nets_str = getenv("ZEEK_LOCAL_NETS"); -global disable_spicy_dhcp = (getenv("ZEEK_DISABLE_SPICY_DHCP") == "") ? F : T; -global disable_spicy_dns = (getenv("ZEEK_DISABLE_SPICY_DNS") == "") ? F : T; -global disable_spicy_http = (getenv("ZEEK_DISABLE_SPICY_HTTP") == "") ? F : T; -global disable_spicy_ipsec = (getenv("ZEEK_DISABLE_SPICY_IPSEC") == "") ? F : T; -global disable_spicy_ldap = (getenv("ZEEK_DISABLE_SPICY_LDAP") == "") ? F : T; -global disable_spicy_openvpn = (getenv("ZEEK_DISABLE_SPICY_OPENVPN") == "") ? F : T; -global disable_spicy_quic = (getenv("ZEEK_DISABLE_SPICY_QUIC") == "") ? F : T; -global disable_spicy_stun = (getenv("ZEEK_DISABLE_SPICY_STUN") == "") ? F : T; -global disable_spicy_tailscale = (getenv("ZEEK_DISABLE_SPICY_TAILSCALE") == "") ? F : T; -global disable_spicy_tftp = (getenv("ZEEK_DISABLE_SPICY_TFTP") == "") ? F : T; -global disable_spicy_wireguard = (getenv("ZEEK_DISABLE_SPICY_WIREGUARD") == "") ? F : T; +global disable_spicy_dhcp = (getenv("ZEEK_DISABLE_SPICY_DHCP") == true_regex) ? T : F; +global disable_spicy_dns = (getenv("ZEEK_DISABLE_SPICY_DNS") == true_regex) ? T : F; +global disable_spicy_http = (getenv("ZEEK_DISABLE_SPICY_HTTP") == true_regex) ? T : F; +global disable_spicy_ipsec = (getenv("ZEEK_DISABLE_SPICY_IPSEC") == true_regex) ? T : F; +global disable_spicy_ldap = (getenv("ZEEK_DISABLE_SPICY_LDAP") == true_regex) ? T : F; +global disable_spicy_openvpn = (getenv("ZEEK_DISABLE_SPICY_OPENVPN") == true_regex) ? T : F; +global disable_spicy_quic = (getenv("ZEEK_DISABLE_SPICY_QUIC") == true_regex) ? T : F; +global disable_spicy_stun = (getenv("ZEEK_DISABLE_SPICY_STUN") == true_regex) ? T : F; +global disable_spicy_tailscale = (getenv("ZEEK_DISABLE_SPICY_TAILSCALE") == true_regex) ? T : F; +global disable_spicy_tftp = (getenv("ZEEK_DISABLE_SPICY_TFTP") == true_regex) ? T : F; +global disable_spicy_wireguard = (getenv("ZEEK_DISABLE_SPICY_WIREGUARD") == true_regex) ? T : F; -global disable_ics_all = (getenv("ZEEK_DISABLE_ICS_ALL") == "") ? F : T; -global disable_ics_bacnet = (getenv("ZEEK_DISABLE_ICS_BACNET") == "") ? F : T; -global disable_ics_bsap = (getenv("ZEEK_DISABLE_ICS_BSAP") == "") ? F : T; -global disable_ics_dnp3 = (getenv("ZEEK_DISABLE_ICS_DNP3") == "") ? F : T; -global disable_ics_enip = (getenv("ZEEK_DISABLE_ICS_ENIP") == "") ? F : T; -global disable_ics_ethercat = (getenv("ZEEK_DISABLE_ICS_ETHERCAT") == "") ? F : T; -global disable_ics_genisys = (getenv("ZEEK_DISABLE_ICS_GENISYS") == "") ? F : T; -global disable_ics_opcua_binary = (getenv("ZEEK_DISABLE_ICS_OPCUA_BINARY") == "") ? F : T; -global disable_ics_modbus = (getenv("ZEEK_DISABLE_ICS_MODBUS") == "") ? F : T; -global disable_ics_profinet = (getenv("ZEEK_DISABLE_ICS_PROFINET") == "") ? F : T; -global disable_ics_s7comm = (getenv("ZEEK_DISABLE_ICS_S7COMM") == "") ? F : T; -global disable_ics_synchrophasor = (getenv("ZEEK_DISABLE_ICS_SYNCHROPHASOR") == "") ? F : T; +global disable_ics_all = (getenv("ZEEK_DISABLE_ICS_ALL") == true_regex) ? T : F; +global disable_ics_bacnet = (getenv("ZEEK_DISABLE_ICS_BACNET") == true_regex) ? T : F; +global disable_ics_bsap = (getenv("ZEEK_DISABLE_ICS_BSAP") == true_regex) ? T : F; +global disable_ics_dnp3 = (getenv("ZEEK_DISABLE_ICS_DNP3") == true_regex) ? T : F; +global disable_ics_enip = (getenv("ZEEK_DISABLE_ICS_ENIP") == true_regex) ? T : F; +global disable_ics_ethercat = (getenv("ZEEK_DISABLE_ICS_ETHERCAT") == true_regex) ? T : F; +global disable_ics_genisys = (getenv("ZEEK_DISABLE_ICS_GENISYS") == true_regex) ? T : F; +global disable_ics_opcua_binary = (getenv("ZEEK_DISABLE_ICS_OPCUA_BINARY") == true_regex) ? T : F; +global disable_ics_modbus = (getenv("ZEEK_DISABLE_ICS_MODBUS") == true_regex) ? T : F; +global disable_ics_profinet = (getenv("ZEEK_DISABLE_ICS_PROFINET") == true_regex) ? T : F; +global disable_ics_s7comm = (getenv("ZEEK_DISABLE_ICS_S7COMM") == true_regex) ? T : F; +global disable_ics_synchrophasor = (getenv("ZEEK_DISABLE_ICS_SYNCHROPHASOR") == true_regex) ? T : F; redef Broker::default_listen_address = "127.0.0.1"; redef ignore_checksums = T; From bf232c65415dcf79483d7b247e813b40dc4ddac4 Mon Sep 17 00:00:00 2001 From: SG Date: Fri, 16 Feb 2024 14:40:52 -0700 Subject: [PATCH 05/79] for supporting JSON logs from Zeek (idaholab/Malcolm#65); almost certainly broken at this point --- .../scripts/filebeat-process-zeek-folder.sh | 9 +- filebeat/scripts/zeek-log-field-bitmap.py | 160 - filebeat/scripts/zeek-log-fields.json | 262 - logstash/pipelines/output/98_finalize.conf | 1 + logstash/pipelines/zeek/10_zeek_prep.conf | 24 +- logstash/pipelines/zeek/11_zeek_parse.conf | 5657 +++++++++-------- 6 files changed, 2919 insertions(+), 3194 deletions(-) delete mode 100755 filebeat/scripts/zeek-log-field-bitmap.py delete mode 100644 filebeat/scripts/zeek-log-fields.json diff --git a/filebeat/scripts/filebeat-process-zeek-folder.sh b/filebeat/scripts/filebeat-process-zeek-folder.sh index 825e26609..5fc0a024e 100755 --- a/filebeat/scripts/filebeat-process-zeek-folder.sh +++ b/filebeat/scripts/filebeat-process-zeek-folder.sh @@ -18,8 +18,6 @@ LOCKDIR="/tmp/zeek-beats-process-folder" export SCRIPT_DIR="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -export ZEEK_LOG_FIELD_BITMAP_SCRIPT="$SCRIPT_DIR/zeek-log-field-bitmap.py" - export ZEEK_LOG_AUTO_TAG=${AUTO_TAG:-"true"} ZEEK_LOGS_DIR=${FILEBEAT_ZEEK_DIR:-/zeek/} @@ -81,13 +79,8 @@ if mkdir $LOCKDIR; then do PROCESS_TIME=$(date +%s%N) TAGS_JOINED=$(printf "%s," "${TAGS[@]}")${PROCESS_TIME} - FIELDS_BITMAP="$($ZEEK_LOG_FIELD_BITMAP_SCRIPT "$LOGFILE" | head -n 1)" LINKNAME_BASE="$(basename "$LOGFILE" .log)" - if [[ -n $FIELDS_BITMAP ]]; then - LINKNAME="${LINKNAME_BASE}(${TAGS_JOINED},${FIELDS_BITMAP}).log" - else - LINKNAME="${LINKNAME_BASE}(${TAGS_JOINED}).log" - fi + LINKNAME="${LINKNAME_BASE}(${TAGS_JOINED}).log" touch "$LOGFILE" ln -sfr "$LOGFILE" "$LINKDIR/$LINKNAME" done diff --git a/filebeat/scripts/zeek-log-field-bitmap.py b/filebeat/scripts/zeek-log-field-bitmap.py deleted file mode 100755 index 6ee960c40..000000000 --- a/filebeat/scripts/zeek-log-field-bitmap.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. - -################################################################################################### -# parse the fields names from the header of of the log file and compare them to the -# known list of total fields. if this zeek log has is a subset of the known fields, -# create a bitmap of the included fields to be included as a special tag -# which can help the logstash parser know on a line-by-line basis which fields are included. -# when logstash-filter-dissect gets this implemented, we may not have to do this: -# - https://github.com/logstash-plugins/logstash-filter-dissect/issues/56 -# - https://github.com/logstash-plugins/logstash-filter-dissect/issues/62 -# -# arguments: accepts one argument, the name of a zeek log file -# output: returns a string suitable for use as a tag indicating the field bitset., eg., ZEEKFLDx00x01FFFFFF -# -# ZEEKFLDx00x01FFFFFF -# | └ bitmap of included fields within field list -# └ index into zeekLogFields list indicating (to support legacy field configurations, see below) -# -# example: -# $ ./zeek-log-field-bitmap.py /path/to/conn.log -# ZEEKFLDx00x01FFFFFF -# -# there are two cases we're trying to cover here by indicating the field types: -# 1. certain fields can be turned on/off in config (for example, enabling/disabling MACs or VLANs for conn.log) -# 2. a Zeek version upgrade changed the field list (see notes about DHCP.log in -# https://docs.zeek.org/en/latest/install/release-notes.html#bro-2-6) -# -# The first case is pretty simple, because in that case the fields in the zeek log will be some subset of -# the list of all known fields for that type. -# -# The second case is more complicated because the field list could be completely different. Because of this case -# each of the entries in zeekLogFields is itself a list, with older configurations occuring earlier in the list -# -# $ zeek-log-field-bitmap.py ./bro2.5/dhcp.log -# ZEEKFLDx00x000003FF -# -# $ zeek-log-field-bitmap.py ./bro2.6/dhcp.log -# ZEEKFLDx01x00007FFF -# - -import sys -import os -import json -from collections import defaultdict -from ordered_set import OrderedSet - -# lists of all known fields for each type of zeek log we're concerned with mapping (ordered as in the .log file header) -# are stored in zeek-log-fields.json -FIELDS_JSON_FILE = os.path.join(os.path.dirname(os.path.realpath(__file__)), "zeek-log-fields.json") - -ZEEK_LOG_DELIMITER = '\t' # zeek log file field delimiter -ZEEK_LOG_HEADER_LOGTYPE = 'path' # header value for zeek log type (conn, weird, etc.) -ZEEK_LOG_HEADER_FIELDS = 'fields' # header value for zeek log fields list - -# file prefix for bitmap to stdout, eg., ZEEKFLDx00x01FFFFFF -ZEEK_LOG_BITMAP_PREFIX = 'ZEEKFLD' - - -################################################################################################### -# print to stderr -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - - -################################################################################################### -# Set the index'th bit of v to 1 if x is truthy, else to 0, and return the new value -def set_bit(v, index, x): - mask = 1 << index # Compute mask, an integer with just bit 'index' set. - v &= ~mask # Clear the bit indicated by the mask (if x is False) - if x: - v |= mask # If x was True, set the bit indicated by the mask. - return v - - -################################################################################################### -# main -def main(): - errCode = os.EX_DATAERR - - dataError = False - zeekLogFields = defaultdict(list) - - # load from json canonical list of known zeek log fields we're concerned with mapping - zeekLogFieldsTmp = json.load(open(FIELDS_JSON_FILE, 'r')) - if isinstance(zeekLogFieldsTmp, dict): - for logType, listOfFieldLists in zeekLogFieldsTmp.items(): - if isinstance(logType, str) and isinstance(listOfFieldLists, list): - zeekLogFields[str(logType)] = [OrderedSet(fieldList) for fieldList in listOfFieldLists] - else: - dataError = True - break - else: - dataError = True - - if dataError: - # something is wrong with the json file - eprint("Error loading {} (not found or incorrectly formatted)".format(FIELDS_JSON_FILE)) - - else: - if (len(sys.argv) == 2) and os.path.isfile(sys.argv[1]): - - fieldsBitmap = 0 - - # loop over header lines in zeek log file (beginning with '#') and extract the header values - # into a dictionary containing, among other things: - # - the "path" which is the zeek log type (eg., conn, weird, etc.) - # - the "fields" list of field names - headers = {} - try: - with open(sys.argv[1], "r", encoding='utf-8') as zeekLogFile: - for line in zeekLogFile: - if line.startswith('#'): - values = line.strip().split(ZEEK_LOG_DELIMITER) - key = values.pop(0)[1:] - if len(values) == 1: - headers[key] = values[0] - else: - headers[key] = values - else: - break - except Exception as e: - eprint("{} for '{}': {}".format(type(e).__name__, sys.argv[1], e)) - - if ( - (ZEEK_LOG_HEADER_LOGTYPE in headers) - and (ZEEK_LOG_HEADER_FIELDS in headers) # the "path" header exists - and (headers[ZEEK_LOG_HEADER_LOGTYPE] in zeekLogFields) # the "fields" header exists - ): # this zeek log type is one we're concerned with mapping - - # the set of field names in *this* log file - logFieldNames = OrderedSet(headers[ZEEK_LOG_HEADER_FIELDS]) - - for versionIdx, allFieldNames in reversed( - list(enumerate(zeekLogFields[headers[ZEEK_LOG_HEADER_LOGTYPE]])) - ): - - # are this logfile's fields a subset of the complete list? - if logFieldNames.issubset(allFieldNames): - - # determine which fields in the complete list are included in this log file - for i, fName in enumerate(allFieldNames): - fieldsBitmap = set_bit(fieldsBitmap, i, fName in logFieldNames) - - # eprint(fieldsBitmap) - print('{0}x{1:02X}x{2:08X}'.format(ZEEK_LOG_BITMAP_PREFIX, versionIdx, fieldsBitmap)) - errCode = os.EX_OK - - else: - # invalid command-line arguments - eprint("{} ".format(sys.argv[0])) - errCode = os.EX_USAGE - - return errCode - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/filebeat/scripts/zeek-log-fields.json b/filebeat/scripts/zeek-log-fields.json deleted file mode 100644 index 290ba4bde..000000000 --- a/filebeat/scripts/zeek-log-fields.json +++ /dev/null @@ -1,262 +0,0 @@ -{ - "conn": [ - [ - "ts", - "uid", - "id.orig_h", - "id.orig_p", - "id.resp_h", - "id.resp_p", - "proto", - "service", - "duration", - "orig_bytes", - "resp_bytes", - "conn_state", - "local_orig", - "local_resp", - "missed_bytes", - "history", - "orig_pkts", - "orig_ip_bytes", - "resp_pkts", - "resp_ip_bytes", - "tunnel_parents", - "vlan", - "inner_vlan", - "orig_l2_addr", - "resp_l2_addr", - "community_id" - ] - ], - "dhcp": [ - [ - "ts", - "uids", - "client_addr", - "server_addr", - "mac", - "host_name", - "client_fqdn", - "domain", - "requested_addr", - "assigned_addr", - "lease_time", - "client_message", - "server_message", - "msg_types", - "duration", - "client_software", - "server_software" - ] - ], - "files": [ - [ - "ts", - "fuid", - "tx_hosts", - "rx_hosts", - "conn_uids", - "source", - "depth", - "analyzers", - "mime_type", - "filename", - "duration", - "local_orig", - "is_orig", - "seen_bytes", - "total_bytes", - "missing_bytes", - "overflow_bytes", - "timedout", - "parent_fuid", - "md5", - "sha1", - "sha256", - "extracted", - "extracted_cutoff", - "extracted_size", - "ftime" - ], - [ - "ts", - "fuid", - "uid", - "id.orig_h", - "id.orig_p", - "id.resp_h", - "id.resp_p", - "source", - "depth", - "analyzers", - "mime_type", - "filename", - "duration", - "local_orig", - "is_orig", - "seen_bytes", - "total_bytes", - "missing_bytes", - "overflow_bytes", - "timedout", - "parent_fuid", - "md5", - "sha1", - "sha256", - "extracted", - "extracted_cutoff", - "extracted_size", - "ftime" - ] - ], - "http": [ - [ - "ts", - "uid", - "id.orig_h", - "id.orig_p", - "id.resp_h", - "id.resp_p", - "trans_depth", - "method", - "host", - "uri", - "referrer", - "version", - "user_agent", - "origin", - "request_body_len", - "response_body_len", - "status_code", - "status_msg", - "info_code", - "info_msg", - "tags", - "username", - "password", - "proxied", - "orig_fuids", - "orig_filenames", - "orig_mime_types", - "resp_fuids", - "resp_filenames", - "resp_mime_types", - "post_username", - "post_password_plain", - "post_password_md5", - "post_password_sha1", - "post_password_sha256" - ] - ], - "rdp": [ - [ - "ts", - "uid", - "id.orig_h", - "id.orig_p", - "id.resp_h", - "id.resp_p", - "cookie", - "result", - "security_protocol", - "client_channels", - "keyboard_layout", - "client_build", - "client_name", - "client_dig_product_id", - "desktop_width", - "desktop_height", - "requested_color_depth", - "cert_type", - "cert_count", - "cert_permanent", - "encryption_level", - "encryption_method" - ] - ], - "smb_files": [ - [ - "ts", - "uid", - "id.orig_h", - "id.orig_p", - "id.resp_h", - "id.resp_p", - "fuid", - "action", - "path", - "name", - "size", - "prev_name", - "times.modified", - "times.accessed", - "times.created", - "times.changed", - "data_offset_req", - "data_len_req", - "data_len_rsp" - ] - ], - "ssh": [ - [ - "ts", - "uid", - "id.orig_h", - "id.orig_p", - "id.resp_h", - "id.resp_p", - "version", - "auth_success", - "auth_attempts", - "direction", - "client", - "server", - "cipher_alg", - "mac_alg", - "compression_alg", - "kex_alg", - "host_key_alg", - "host_key", - "remote_location.country_code", - "remote_location.region", - "remote_location.city", - "remote_location.latitude", - "remote_location.longitude", - "hasshVersion", - "hassh", - "hasshServer", - "cshka", - "hasshAlgorithms", - "sshka", - "hasshServerAlgorithms" - ] - ], - "ssl": [ - [ - "ts", - "uid", - "id.orig_h", - "id.orig_p", - "id.resp_h", - "id.resp_p", - "version", - "cipher", - "curve", - "server_name", - "resumed", - "last_alert", - "next_protocol", - "established", - "ssl_history", - "cert_chain_fps", - "client_cert_chain_fps", - "sni_matches_cert", - "validation_status", - "ja3", - "ja3s" - ] - ] -} - - diff --git a/logstash/pipelines/output/98_finalize.conf b/logstash/pipelines/output/98_finalize.conf index 6cf43e49e..4a2580c64 100644 --- a/logstash/pipelines/output/98_finalize.conf +++ b/logstash/pipelines/output/98_finalize.conf @@ -20,6 +20,7 @@ filter { "_geoip_lookup_failure", "_grokparsefailure", "_jsonparsefailure", + "_jsonparsesuccess", "_malcolm_miscbeat", "_malcolm_beats", "_ouilookupfailure", diff --git a/logstash/pipelines/zeek/10_zeek_prep.conf b/logstash/pipelines/zeek/10_zeek_prep.conf index 5201f75a7..da3ebf627 100644 --- a/logstash/pipelines/zeek/10_zeek_prep.conf +++ b/logstash/pipelines/zeek/10_zeek_prep.conf @@ -38,31 +38,9 @@ filter { id => "ruby_zeek_prune_tags" code => " filenameTags = event.get('[@metadata][zeek_log_tags]').split(',') - zeekFieldsTag = filenameTags.select { |str| str.start_with?('ZEEKFLDx') } - if (zeekFieldsTag.size > 0) then - zeekFieldsTags = zeekFieldsTag.first.split('x', 3) - if (zeekFieldsTags.size == 3) then - event.set('[@metadata][zeek_fields_bitmap_version]', zeekFieldsTags[1].to_i(16)) - event.set('[@metadata][zeek_fields_bitmap]', zeekFieldsTags[2].to_i(16)) - end - end - filenameTags.delete_if{|v| ((v == nil) or (v == '') or (v !~ /\D/) or (v =~ /\A\s*(ZEEKFLDx|autocarve)/i) or (v =~ /\A\s*(pcap|dmp|log|bro|zeek|suricata|m?tcpdump|m?netsniff|autozeek|autosuricata)s?\s*\z/i) or (v == event.get('[log_source]')))} + filenameTags.delete_if{|v| ((v == nil) or (v == '') or (v !~ /\D/) or (v =~ /\A\s*(autocarve)/i) or (v =~ /\A\s*(pcap|dmp|log|bro|zeek|suricata|m?tcpdump|m?netsniff|autozeek|autosuricata)s?\s*\z/i) or (v == event.get('[log_source]')))} event.set('[@metadata][zeek_log_tags]', filenameTags.uniq) unless (filenameTags.length == 0) " - # - # ZEEKFLDx00x01FFFFFF - # | | └ bitmap of included fields within field list - # | └ index into zeek-log-field-bitmap.py:ZEEK_LOG_FIELDS list indicating field configuration within differing Zeek versions - # └ indicates that the field list has been pre-processed by zeek-process-pcap.py - # - # when logstash-filter-dissect gets this implemented, we *may* not have to do this - # - see zeek-process-pcap.py for the format of the bitmap number - # - https://github.com/logstash-plugins/logstash-filter-dissect/issues/56 - # - https://github.com/logstash-plugins/logstash-filter-dissect/issues/62 - # - # todo: right now rather than using the bitmap intelligently to build the field list, I'm just looking for - # known preconfigurations. look into doing it "smart". - # } if ([@metadata][zeek_log_tags]) { mutate { id => "mutate_merge_zeek_log_tags" merge => { "[tags]" => "[@metadata][zeek_log_tags]" } } } diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index 6cec9e898..ed6dfda3a 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -1,4 +1,5 @@ -######################## # zeek -> arkime session creation and enrichment +######################## +# zeek -> arkime session creation and enrichment # # see https://docs.zeek.org/en/stable/script-reference/log-files.html for Zeek logfile documentation # @@ -15,6 +16,14 @@ filter { + # handle JSON-formatted Zeek logs right out of the gate, we'll do the field renaming below + if ([message] =~ /^{.*}$/) { json { + id => "json_zeek_message_parse" + source => "[message]" + target => "[zeek_cols]" + add_tag => [ "_jsonparsesuccess" ] + } } + # The Dissect is WAY faster than CSV, and quite a bit faster than mutate.split. However, it # is not as flexible when it comes to missing or extra columns # (See https://github.com/logstash-plugins/logstash-filter-dissect/issues/62) @@ -31,131 +40,27 @@ filter { # conn.log # https://docs.zeek.org/en/stable/scripts/base/protocols/conn/main.zeek.html#type-Conn::Info - if ([@metadata][zeek_fields_bitmap] and [@metadata][zeek_fields_bitmap_version]) { - - # bitmap conn.log field configuration version 0 - # - # Todo: I need to just bite the bullet and code this up in a Ruby block rather than hardcoding values. - # - # vlan + mac + community_id: 0x3ffffff / 67108863 - # vlan + mac: 0x1ffffff / 33554431 - # mac + community_id: 0x39fffff / 60817407 - # mac only: 0x19fffff / 27262975 - # vlan + community_id: 0x27fffff / 41943039 - # vlan only: 0x7fffff / 8388607 - # community_id only: 0x1fffff / 2097151 - # no custom fields: - - if ([@metadata][zeek_fields_bitmap_version] == 0) { - - if ([@metadata][zeek_fields_bitmap] == 67108863) { - # conn.log has vlans + macs + community_id - dissect { - id => "dissect_zeek_conn_with_all_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]} %{[zeek_cols][vlan]} %{[zeek_cols][inner_vlan]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][community_id]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 33554431) { - # conn.log has vlans + macs - dissect { - id => "dissect_zeek_conn_with_vlan_and_mac_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]} %{[zeek_cols][vlan]} %{[zeek_cols][inner_vlan]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 60817407) { - # conn.log has macs + community_id - dissect { - id => "dissect_zeek_conn_with_mac_and_community_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][community_id]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 27262975) { - # conn.log has macs - dissect { - id => "dissect_zeek_conn_with_mac_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 41943039) { - # conn.log has vlans + community_id - dissect { - id => "dissect_zeek_conn_with_vlan_and_community_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]} %{[zeek_cols][vlan]} %{[zeek_cols][inner_vlan]} %{[zeek_cols][community_id]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 8388607) { - # conn.log has vlans - dissect { - id => "dissect_zeek_conn_with_vlan_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]} %{[zeek_cols][vlan]} %{[zeek_cols][inner_vlan]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 2097151) { - # conn.log has community_id - dissect { - id => "dissect_zeek_conn_with_community_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]} %{[zeek_cols][community_id]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 2097151) { - # conn.log has no custom fields - dissect { - id => "dissect_zeek_conn_with_minimal_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]}" - } - } - - } else { - # who knows? the conn.log preprocessed bitmap is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_conn_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - } else { - # who knows? the conn.log preprocessed bitmap field list version is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_bitmap_conn_version" - add_tag => [ "_dissectfailure" ] } - } + if "_jsonparsesuccess" in [tags] { } else { - # who knows? the conn.log was not preprocessed to determine fields, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_missing_conn_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_conn" + dissect { + id => "dissect_zeek_conn_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]} %{[zeek_cols][vlan]} %{[zeek_cols][inner_vlan]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][community_id]}" + } } - ruby { - id => "ruby_zip_zeek_conn" - init => "@zeek_conn_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'service', 'duration', 'orig_bytes', 'resp_bytes', 'conn_state', 'local_orig', 'local_resp', 'missed_bytes', 'history', 'orig_pkts', 'orig_ip_bytes', 'resp_pkts', 'resp_ip_bytes', 'tunnel_parents', 'vlan', 'inner_vlan', 'orig_l2_addr', 'resp_l2_addr', 'community_id' ]" - code => "event.set('[zeek_cols]', @zeek_conn_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_conn" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_conn" + init => "@zeek_conn_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'service', 'duration', 'orig_bytes', 'resp_bytes', 'conn_state', 'local_orig', 'local_resp', 'missed_bytes', 'history', 'orig_pkts', 'orig_ip_bytes', 'resp_pkts', 'resp_ip_bytes', 'tunnel_parents', 'vlan', 'inner_vlan', 'orig_l2_addr', 'resp_l2_addr', 'community_id' ]" + code => "event.set('[zeek_cols]', @zeek_conn_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -210,23 +115,27 @@ filter { # bacnet.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_bacnet" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][bvlc_function]} %{[zeek_cols][pdu_type]} %{[zeek_cols][pdu_service]} %{[zeek_cols][invoke_id]} %{[zeek_cols][result_code]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bacnet" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_bacnet" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][bvlc_function]} %{[zeek_cols][pdu_type]} %{[zeek_cols][pdu_service]} %{[zeek_cols][invoke_id]} %{[zeek_cols][result_code]}" + } } - ruby { - id => "ruby_zip_zeek_bacnet" - init => "@zeek_bacnet_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'bvlc_function', 'pdu_type', 'pdu_service', 'invoke_id', 'result_code' ]" - code => "event.set('[zeek_cols]', @zeek_bacnet_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bacnet" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bacnet" + init => "@zeek_bacnet_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'bvlc_function', 'pdu_type', 'pdu_service', 'invoke_id', 'result_code' ]" + code => "event.set('[zeek_cols]', @zeek_bacnet_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -243,23 +152,27 @@ filter { ############################################################################################################################# # bestguess.log - dissect { - id => "dissect_zeek_bestguess" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][name]} %{[zeek_cols][category]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bestguess" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_bestguess" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][name]} %{[zeek_cols][category]}" + } } - ruby { - id => "ruby_zip_zeek_bestguess" - init => "@zeek_bestguess_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'name', 'category' ]" - code => "event.set('[zeek_cols]', @zeek_bestguess_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bestguess" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bestguess" + init => "@zeek_bestguess_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'name', 'category' ]" + code => "event.set('[zeek_cols]', @zeek_bestguess_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -271,23 +184,27 @@ filter { # bsap_ip_header.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_bsap_ip_header" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][num_msg]} %{[zeek_cols][type_name]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bsap_ip_header" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_bsap_ip_header" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][num_msg]} %{[zeek_cols][type_name]}" + } } - ruby { - id => "ruby_zip_zeek_bsap_ip_header" - init => "@zeek_bsap_ip_header_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'num_msg', 'type_name' ]" - code => "event.set('[zeek_cols]', @zeek_bsap_ip_header_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bsap_ip_header" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bsap_ip_header" + init => "@zeek_bsap_ip_header_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'num_msg', 'type_name' ]" + code => "event.set('[zeek_cols]', @zeek_bsap_ip_header_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -305,23 +222,27 @@ filter { # bsap_ip_rdb.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_bsap_ip_rdb" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][header_size]} %{[zeek_cols][mes_seq]} %{[zeek_cols][res_seq]} %{[zeek_cols][data_len]} %{[zeek_cols][sequence]} %{[zeek_cols][app_func_code]} %{[zeek_cols][node_status]} %{[zeek_cols][func_code]} %{[zeek_cols][variable_count]} %{[zeek_cols][variables]} %{[zeek_cols][variable_value]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bsap_ip_rdb" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_bsap_ip_rdb" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][header_size]} %{[zeek_cols][mes_seq]} %{[zeek_cols][res_seq]} %{[zeek_cols][data_len]} %{[zeek_cols][sequence]} %{[zeek_cols][app_func_code]} %{[zeek_cols][node_status]} %{[zeek_cols][func_code]} %{[zeek_cols][variable_count]} %{[zeek_cols][variables]} %{[zeek_cols][variable_value]}" + } } - ruby { - id => "ruby_zip_zeek_bsap_ip_rdb" - init => "@zeek_bsap_ip_rdb_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'header_size', 'mes_seq', 'res_seq', 'data_len', 'sequence', 'app_func_code', 'node_status', 'func_code', 'variable_count', 'variables', 'variable_value' ]" - code => "event.set('[zeek_cols]', @zeek_bsap_ip_rdb_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bsap_ip_rdb" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bsap_ip_rdb" + init => "@zeek_bsap_ip_rdb_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'header_size', 'mes_seq', 'res_seq', 'data_len', 'sequence', 'app_func_code', 'node_status', 'func_code', 'variable_count', 'variables', 'variable_value' ]" + code => "event.set('[zeek_cols]', @zeek_bsap_ip_rdb_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -339,23 +260,27 @@ filter { # bsap_serial_header.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_bsap_serial_header" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][ser]} %{[zeek_cols][dadd]} %{[zeek_cols][sadd]} %{[zeek_cols][ctl]} %{[zeek_cols][dfun]} %{[zeek_cols][seq]} %{[zeek_cols][sfun]} %{[zeek_cols][nsb]} %{[zeek_cols][type_name]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bsap_serial_header" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_bsap_serial_header" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][ser]} %{[zeek_cols][dadd]} %{[zeek_cols][sadd]} %{[zeek_cols][ctl]} %{[zeek_cols][dfun]} %{[zeek_cols][seq]} %{[zeek_cols][sfun]} %{[zeek_cols][nsb]} %{[zeek_cols][type_name]}" + } } - ruby { - id => "ruby_zip_zeek_bsap_serial_header" - init => "@zeek_bsap_serial_header_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'ser', 'dadd', 'sadd', 'ctl', 'dfun', 'seq', 'sfun', 'nsb', 'type_name' ]" - code => "event.set('[zeek_cols]', @zeek_bsap_serial_header_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bsap_serial_header" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bsap_serial_header" + init => "@zeek_bsap_serial_header_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'ser', 'dadd', 'sadd', 'ctl', 'dfun', 'seq', 'sfun', 'nsb', 'type_name' ]" + code => "event.set('[zeek_cols]', @zeek_bsap_serial_header_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -373,23 +298,27 @@ filter { # bsap_serial_rdb.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_bsap_serial_rdb" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][func_code]} %{[zeek_cols][variable_count]} %{[zeek_cols][variables]} %{[zeek_cols][variable_value]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bsap_serial_rdb" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_bsap_serial_rdb" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][func_code]} %{[zeek_cols][variable_count]} %{[zeek_cols][variables]} %{[zeek_cols][variable_value]}" + } } - ruby { - id => "ruby_zip_zeek_bsap_serial_rdb" - init => "@zeek_bsap_serial_rdb_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'func_code', 'variable_count', 'variables', 'variable_value' ]" - code => "event.set('[zeek_cols]', @zeek_bsap_serial_rdb_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bsap_serial_rdb" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bsap_serial_rdb" + init => "@zeek_bsap_serial_rdb_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'func_code', 'variable_count', 'variables', 'variable_value' ]" + code => "event.set('[zeek_cols]', @zeek_bsap_serial_rdb_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -407,23 +336,27 @@ filter { # bsap_serial_rdb_ext.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_bsap_serial_rdb_ext" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][dfun]} %{[zeek_cols][seq]} %{[zeek_cols][sfun]} %{[zeek_cols][nsb]} %{[zeek_cols][extfun]} %{[zeek_cols][data]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bsap_serial_rdb_ext" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_bsap_serial_rdb_ext" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][dfun]} %{[zeek_cols][seq]} %{[zeek_cols][sfun]} %{[zeek_cols][nsb]} %{[zeek_cols][extfun]} %{[zeek_cols][data]}" + } } - ruby { - id => "ruby_zip_zeek_bsap_serial_rdb_ext" - init => "@zeek_bsap_serial_rdb_ext_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'dfun', 'seq', 'sfun', 'nsb', 'extfun', 'data' ]" - code => "event.set('[zeek_cols]', @zeek_bsap_serial_rdb_ext_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bsap_serial_rdb_ext" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bsap_serial_rdb_ext" + init => "@zeek_bsap_serial_rdb_ext_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'dfun', 'seq', 'sfun', 'nsb', 'extfun', 'data' ]" + code => "event.set('[zeek_cols]', @zeek_bsap_serial_rdb_ext_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -447,23 +380,27 @@ filter { # bacnet_device_control.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_bacnet_device_control" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][invoke_id]} %{[zeek_cols][pdu_service]} %{[zeek_cols][time_duration]} %{[zeek_cols][device_state]} %{[zeek_cols][password]} %{[zeek_cols][result]} %{[zeek_cols][result_code]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bacnet_device_control" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_bacnet_device_control" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][invoke_id]} %{[zeek_cols][pdu_service]} %{[zeek_cols][time_duration]} %{[zeek_cols][device_state]} %{[zeek_cols][password]} %{[zeek_cols][result]} %{[zeek_cols][result_code]}" + } } - ruby { - id => "ruby_zip_zeek_bacnet_device_control" - init => "@zeek_bacnet_device_control_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'invoke_id', 'pdu_service', 'time_duration', 'device_state', 'password', 'result', 'result_code' ]" - code => "event.set('[zeek_cols]', @zeek_bacnet_device_control_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bacnet_device_control" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bacnet_device_control" + init => "@zeek_bacnet_device_control_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'invoke_id', 'pdu_service', 'time_duration', 'device_state', 'password', 'result', 'result_code' ]" + code => "event.set('[zeek_cols]', @zeek_bacnet_device_control_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -481,23 +418,27 @@ filter { # bacnet_discovery.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_bacnet_discovery" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][pdu_service]} %{[zeek_cols][object_type]} %{[zeek_cols][instance_number]} %{[zeek_cols][vendor]} %{[zeek_cols][range]} %{[zeek_cols][object_name]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bacnet_discovery" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_bacnet_discovery" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][pdu_service]} %{[zeek_cols][object_type]} %{[zeek_cols][instance_number]} %{[zeek_cols][vendor]} %{[zeek_cols][range]} %{[zeek_cols][object_name]}" + } } - ruby { - id => "ruby_zip_zeek_bacnet_discovery" - init => "@zeek_bacnet_discovery_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'pdu_service', 'object_type', 'instance_number', 'vendor', 'range', 'object_name' ]" - code => "event.set('[zeek_cols]', @zeek_bacnet_discovery_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bacnet_discovery" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bacnet_discovery" + init => "@zeek_bacnet_discovery_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'pdu_service', 'object_type', 'instance_number', 'vendor', 'range', 'object_name' ]" + code => "event.set('[zeek_cols]', @zeek_bacnet_discovery_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -515,23 +456,27 @@ filter { # bacnet_property.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_bacnet_property" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][invoke_id]} %{[zeek_cols][pdu_service]} %{[zeek_cols][object_type]} %{[zeek_cols][instance_number]} %{[zeek_cols][property]} %{[zeek_cols][array_index]} %{[zeek_cols][value]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bacnet_property" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_bacnet_property" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][invoke_id]} %{[zeek_cols][pdu_service]} %{[zeek_cols][object_type]} %{[zeek_cols][instance_number]} %{[zeek_cols][property]} %{[zeek_cols][array_index]} %{[zeek_cols][value]}" + } } - ruby { - id => "ruby_zip_zeek_bacnet_property" - init => "@zeek_bacnet_property_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'invoke_id', 'pdu_service', 'object_type', 'instance_number', 'property', 'array_index', 'value' ]" - code => "event.set('[zeek_cols]', @zeek_bacnet_property_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bacnet_property" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bacnet_property" + init => "@zeek_bacnet_property_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'invoke_id', 'pdu_service', 'object_type', 'instance_number', 'property', 'array_index', 'value' ]" + code => "event.set('[zeek_cols]', @zeek_bacnet_property_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -551,23 +496,27 @@ filter { # # todo: class_id, instance_id is a hex integer, should it be converted to an integer? - dissect { - id => "dissect_zeek_cip" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][cip_sequence_count]} %{[zeek_cols][direction]} %{[zeek_cols][cip_service_code]} %{[zeek_cols][cip_service]} %{[zeek_cols][cip_status_code]} %{[zeek_cols][cip_status]} %{[zeek_cols][cip_extended_status_code]} %{[zeek_cols][cip_extended_status]} %{[zeek_cols][class_id]} %{[zeek_cols][class_name]} %{[zeek_cols][instance_id]} %{[zeek_cols][attribute_id]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_cip" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_cip" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][cip_sequence_count]} %{[zeek_cols][direction]} %{[zeek_cols][cip_service_code]} %{[zeek_cols][cip_service]} %{[zeek_cols][cip_status_code]} %{[zeek_cols][cip_status]} %{[zeek_cols][cip_extended_status_code]} %{[zeek_cols][cip_extended_status]} %{[zeek_cols][class_id]} %{[zeek_cols][class_name]} %{[zeek_cols][instance_id]} %{[zeek_cols][attribute_id]}" + } } - ruby { - id => "ruby_zip_zeek_cip" - init => "@zeek_cip_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'cip_sequence_count', 'direction', 'cip_service_code', 'cip_service', 'cip_status_code', 'cip_status', 'cip_extended_status_code', 'cip_extended_status', 'class_id', 'class_name', 'instance_id', 'attribute_id' ]" - code => "event.set('[zeek_cols]', @zeek_cip_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_cip" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_cip" + init => "@zeek_cip_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'cip_sequence_count', 'direction', 'cip_service_code', 'cip_service', 'cip_status_code', 'cip_status', 'cip_extended_status_code', 'cip_extended_status', 'class_id', 'class_name', 'instance_id', 'attribute_id' ]" + code => "event.set('[zeek_cols]', @zeek_cip_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -584,23 +533,27 @@ filter { # # TODO: device_status and device_state are a hex int string, convert to int? - dissect { - id => "dissect_zeek_cip_identity" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][encapsulation_version]} %{[zeek_cols][socket_address]} %{[zeek_cols][socket_port]} %{[zeek_cols][vendor_id]} %{[zeek_cols][vendor_name]} %{[zeek_cols][device_type_id]} %{[zeek_cols][device_type_name]} %{[zeek_cols][product_code]} %{[zeek_cols][revision]} %{[zeek_cols][device_status]} %{[zeek_cols][serial_number]} %{[zeek_cols][product_name]} %{[zeek_cols][device_state]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_cip_identity" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_cip_identity" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][encapsulation_version]} %{[zeek_cols][socket_address]} %{[zeek_cols][socket_port]} %{[zeek_cols][vendor_id]} %{[zeek_cols][vendor_name]} %{[zeek_cols][device_type_id]} %{[zeek_cols][device_type_name]} %{[zeek_cols][product_code]} %{[zeek_cols][revision]} %{[zeek_cols][device_status]} %{[zeek_cols][serial_number]} %{[zeek_cols][product_name]} %{[zeek_cols][device_state]}" + } } - ruby { - id => "ruby_zip_zeek_cip_identity" - init => "@zeek_cip_identity_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'encapsulation_version', 'socket_address', 'socket_port', 'vendor_id', 'vendor_name', 'device_type_id', 'device_type_name', 'product_code', 'device_status', 'serial_number', 'product_name', 'device_state' ]" - code => "event.set('[zeek_cols]', @zeek_cip_identity_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_cip_identity" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_cip_identity" + init => "@zeek_cip_identity_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'encapsulation_version', 'socket_address', 'socket_port', 'vendor_id', 'vendor_name', 'device_type_id', 'device_type_name', 'product_code', 'device_status', 'serial_number', 'product_name', 'device_state' ]" + code => "event.set('[zeek_cols]', @zeek_cip_identity_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -616,23 +569,27 @@ filter { # https://github.com/cisagov/ICSNPP # - dissect { - id => "dissect_zeek_cip_io" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][connection_id]} %{[zeek_cols][sequence_number]} %{[zeek_cols][data_length]} %{[zeek_cols][io_data]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_cip_io" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_cip_io" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][connection_id]} %{[zeek_cols][sequence_number]} %{[zeek_cols][data_length]} %{[zeek_cols][io_data]}" + } } - ruby { - id => "ruby_zip_zeek_cip_io" - init => "@zeek_cip_io_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'connection_id', 'sequence_number', 'data_length', 'io_data' ]" - code => "event.set('[zeek_cols]', @zeek_cip_io_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_cip_io" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_cip_io" + init => "@zeek_cip_io_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'connection_id', 'sequence_number', 'data_length', 'io_data' ]" + code => "event.set('[zeek_cols]', @zeek_cip_io_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -647,23 +604,27 @@ filter { # dce_rpc.log # https://docs.zeek.org/en/stable/scripts/base/protocols/dce-rpc/main.zeek.html#type-DCE_RPC::Info - dissect { - id => "dissect_zeek_dce_rpc" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][rtt]} %{[zeek_cols][named_pipe]} %{[zeek_cols][endpoint]} %{[zeek_cols][operation]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_dce_rpc" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_dce_rpc" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][rtt]} %{[zeek_cols][named_pipe]} %{[zeek_cols][endpoint]} %{[zeek_cols][operation]}" + } } - ruby { - id => "ruby_zip_zeek_dce_rpc" - init => "@zeek_dce_rpc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'rtt', 'named_pipe', 'endpoint', 'operation' ]" - code => "event.set('[zeek_cols]', @zeek_dce_rpc_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_dce_rpc" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_dce_rpc" + init => "@zeek_dce_rpc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'rtt', 'named_pipe', 'endpoint', 'operation' ]" + code => "event.set('[zeek_cols]', @zeek_dce_rpc_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -680,61 +641,27 @@ filter { # dhcp.log # https://docs.zeek.org/en/stable/scripts/base/protocols/dhcp/main.zeek.html#type-DHCP::Info - if ([@metadata][zeek_fields_bitmap] and [@metadata][zeek_fields_bitmap_version]) { - - # bitmap dhcp.log field configuration version 0 - # - # all fields : 0x0001FFFF / 131071 - # all fields minus software: 0x00007FFF / 32767 - - if ([@metadata][zeek_fields_bitmap_version] == 0) { - - if ([@metadata][zeek_fields_bitmap] == 131071) { - dissect { - id => "dissect_zeek_dhcp_with_all_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][resp_h]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][host_name]} %{[zeek_cols][client_fqdn]} %{[zeek_cols][domain]} %{[zeek_cols][requested_ip]} %{[zeek_cols][assigned_ip]} %{[zeek_cols][lease_time]} %{[zeek_cols][client_message]} %{[zeek_cols][server_message]} %{[zeek_cols][msg_types]} %{[zeek_cols][duration]} %{[zeek_cols][client_software]} %{[zeek_cols][server_software]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 32767) { - dissect { - id => "dissect_zeek_dhcp_with_all_fields_minus_software" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][resp_h]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][host_name]} %{[zeek_cols][client_fqdn]} %{[zeek_cols][domain]} %{[zeek_cols][requested_ip]} %{[zeek_cols][assigned_ip]} %{[zeek_cols][lease_time]} %{[zeek_cols][client_message]} %{[zeek_cols][server_message]} %{[zeek_cols][msg_types]} %{[zeek_cols][duration]}" - } - } - - } else { - # who knows? the dhcp.log preprocessed bitmap is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_dhcp_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - } else { - # who knows? the dhcp.log preprocessed bitmap field list version is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_bitmap_dhcp_version" - add_tag => [ "_dissectfailure" ] } - } + if "_jsonparsesuccess" in [tags] { } else { - # who knows? the dhcp.log was not preprocessed to determine fields, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_missing_dhcp_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_dhcp" + dissect { + id => "dissect_zeek_dhcp_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][resp_h]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][host_name]} %{[zeek_cols][client_fqdn]} %{[zeek_cols][domain]} %{[zeek_cols][requested_ip]} %{[zeek_cols][assigned_ip]} %{[zeek_cols][lease_time]} %{[zeek_cols][client_message]} %{[zeek_cols][server_message]} %{[zeek_cols][msg_types]} %{[zeek_cols][duration]} %{[zeek_cols][client_software]} %{[zeek_cols][server_software]}" + } } - ruby { - id => "ruby_zip_zeek_dhcp" - init => "@zeek_dhcp_field_names = [ 'ts', 'uid', 'orig_h', 'resp_h', 'orig_l2_addr', 'host_name', 'client_fqdn', 'domain', 'requested_ip', 'assigned_ip', 'lease_time', 'client_message', 'server_message', 'msg_types', 'duration', 'client_software', 'server_software' ]" - code => "event.set('[zeek_cols]', @zeek_dhcp_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_dhcp" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_dhcp" + init => "@zeek_dhcp_field_names = [ 'ts', 'uid', 'orig_h', 'resp_h', 'orig_l2_addr', 'host_name', 'client_fqdn', 'domain', 'requested_ip', 'assigned_ip', 'lease_time', 'client_message', 'server_message', 'msg_types', 'duration', 'client_software', 'server_software' ]" + code => "event.set('[zeek_cols]', @zeek_dhcp_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -770,23 +697,27 @@ filter { # dnp3.log # https://docs.zeek.org/en/stable/scripts/base/protocols/dnp3/main.zeek.html#type-DNP3::Info - dissect { - id => "dissect_zeek_dnp3" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][fc_request]} %{[zeek_cols][fc_reply]} %{[zeek_cols][iin]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_dnp3" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_dnp3" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][fc_request]} %{[zeek_cols][fc_reply]} %{[zeek_cols][iin]}" + } } - ruby { - id => "ruby_zip_zeek_dnp3" - init => "@zeek_dnp3_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'fc_request', 'fc_reply', 'iin' ]" - code => "event.set('[zeek_cols]', @zeek_dnp3_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_dnp3" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_dnp3" + init => "@zeek_dnp3_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'fc_request', 'fc_reply', 'iin' ]" + code => "event.set('[zeek_cols]', @zeek_dnp3_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -801,23 +732,27 @@ filter { # dnp3_control.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_dnp3_control" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][block_type]} %{[zeek_cols][function_code]} %{[zeek_cols][index_number]} %{[zeek_cols][trip_control_code]} %{[zeek_cols][operation_type]} %{[zeek_cols][execute_count]} %{[zeek_cols][on_time]} %{[zeek_cols][off_time]} %{[zeek_cols][status_code]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_dnp3_control" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_dnp3_control" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][block_type]} %{[zeek_cols][function_code]} %{[zeek_cols][index_number]} %{[zeek_cols][trip_control_code]} %{[zeek_cols][operation_type]} %{[zeek_cols][execute_count]} %{[zeek_cols][on_time]} %{[zeek_cols][off_time]} %{[zeek_cols][status_code]}" + } } - ruby { - id => "ruby_zip_zeek_dnp3_control" - init => "@zeek_dnp3_control_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'block_type', 'function_code', 'index_number', 'trip_control_code', 'operation_type', 'execute_count', 'on_time', 'off_time', 'status_code' ]" - code => "event.set('[zeek_cols]', @zeek_dnp3_control_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_dnp3_control" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_dnp3_control" + init => "@zeek_dnp3_control_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'block_type', 'function_code', 'index_number', 'trip_control_code', 'operation_type', 'execute_count', 'on_time', 'off_time', 'status_code' ]" + code => "event.set('[zeek_cols]', @zeek_dnp3_control_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -832,23 +767,27 @@ filter { # dnp3_objects.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_dnp3_objects" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][function_code]} %{[zeek_cols][object_type]} %{[zeek_cols][object_count]} %{[zeek_cols][range_low]} %{[zeek_cols][range_high]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_dnp3_objects" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_dnp3_objects" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][function_code]} %{[zeek_cols][object_type]} %{[zeek_cols][object_count]} %{[zeek_cols][range_low]} %{[zeek_cols][range_high]}" + } } - ruby { - id => "ruby_zip_zeek_dnp3_objects" - init => "@zeek_dnp3_objects_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'function_code', 'object_type', 'object_count', 'range_low', 'range_high' ]" - code => "event.set('[zeek_cols]', @zeek_dnp3_objects_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_dnp3_objects" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_dnp3_objects" + init => "@zeek_dnp3_objects_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'function_code', 'object_type', 'object_count', 'range_low', 'range_high' ]" + code => "event.set('[zeek_cols]', @zeek_dnp3_objects_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -863,23 +802,27 @@ filter { # dns.log # https://docs.zeek.org/en/stable/scripts/base/protocols/dns/main.zeek.html#type-DNS::Info - dissect { - id => "dissect_zeek_dns" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][trans_id]} %{[zeek_cols][rtt]} %{[zeek_cols][query]} %{[zeek_cols][qclass]} %{[zeek_cols][qclass_name]} %{[zeek_cols][qtype]} %{[zeek_cols][qtype_name]} %{[zeek_cols][rcode]} %{[zeek_cols][rcode_name]} %{[zeek_cols][AA]} %{[zeek_cols][TC]} %{[zeek_cols][RD]} %{[zeek_cols][RA]} %{[zeek_cols][Z]} %{[zeek_cols][answers]} %{[zeek_cols][TTLs]} %{[zeek_cols][rejected]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_dns" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_dns" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][trans_id]} %{[zeek_cols][rtt]} %{[zeek_cols][query]} %{[zeek_cols][qclass]} %{[zeek_cols][qclass_name]} %{[zeek_cols][qtype]} %{[zeek_cols][qtype_name]} %{[zeek_cols][rcode]} %{[zeek_cols][rcode_name]} %{[zeek_cols][AA]} %{[zeek_cols][TC]} %{[zeek_cols][RD]} %{[zeek_cols][RA]} %{[zeek_cols][Z]} %{[zeek_cols][answers]} %{[zeek_cols][TTLs]} %{[zeek_cols][rejected]}" + } } - ruby { - id => "ruby_zip_zeek_dns" - init => "@zeek_dns_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'trans_id', 'rtt', 'query', 'qclass', 'qclass_name', 'qtype', 'qtype_name', 'rcode', 'rcode_name', 'AA', 'TC', 'RD', 'RA', 'Z', 'answers', 'TTLs', 'rejected' ]" - code => "event.set('[zeek_cols]', @zeek_dns_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_dns" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_dns" + init => "@zeek_dns_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'trans_id', 'rtt', 'query', 'qclass', 'qclass_name', 'qtype', 'qtype_name', 'rcode', 'rcode_name', 'AA', 'TC', 'RD', 'RA', 'Z', 'answers', 'TTLs', 'rejected' ]" + code => "event.set('[zeek_cols]', @zeek_dns_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -897,23 +840,27 @@ filter { # dpd.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/dpd/main.zeek.html#type-DPD::Info - dissect { - id => "dissect_zeek_dpd" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][failure_reason]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_dpd" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_dpd" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][failure_reason]}" + } } - ruby { - id => "ruby_zip_zeek_dpd" - init => "@zeek_dpd_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'service', 'failure_reason' ]" - code => "event.set('[zeek_cols]', @zeek_dpd_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_dpd" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_dpd" + init => "@zeek_dpd_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'service', 'failure_reason' ]" + code => "event.set('[zeek_cols]', @zeek_dpd_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -938,23 +885,27 @@ filter { # https://github.com/cisagov/ICSNPP # - dissect { - id => "dissect_zeek_enip" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][enip_command_code]} %{[zeek_cols][enip_command]} %{[zeek_cols][length]} %{[zeek_cols][session_handle]} %{[zeek_cols][enip_status]} %{[zeek_cols][sender_context]} %{[zeek_cols][options]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_enip" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_enip" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][enip_command_code]} %{[zeek_cols][enip_command]} %{[zeek_cols][length]} %{[zeek_cols][session_handle]} %{[zeek_cols][enip_status]} %{[zeek_cols][sender_context]} %{[zeek_cols][options]}" + } } - ruby { - id => "ruby_zip_zeek_enip" - init => "@zeek_enip_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'enip_command', 'length', 'session_handle', 'enip_status', 'sender_context', 'options' ]" - code => "event.set('[zeek_cols]', @zeek_enip_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_enip" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_enip" + init => "@zeek_enip_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'enip_command', 'length', 'session_handle', 'enip_status', 'sender_context', 'options' ]" + code => "event.set('[zeek_cols]', @zeek_enip_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -969,23 +920,27 @@ filter { # ecat_registers.log # https://github.com/cisagov/icsnpp-ethercat - dissect { - id => "dissect_zeek_ecat_registers" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][command]} %{[zeek_cols][server_addr]} %{[zeek_cols][register_type]} %{[zeek_cols][register_addr]} %{[zeek_cols][data]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ecat_registers" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_ecat_registers" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][command]} %{[zeek_cols][server_addr]} %{[zeek_cols][register_type]} %{[zeek_cols][register_addr]} %{[zeek_cols][data]}" + } } - ruby { - id => "ruby_zip_zeek_ecat_registers" - init => "@zeek_ecat_registers_field_names = [ 'ts', 'orig_l2_addr', 'resp_l2_addr', 'command', 'server_addr', 'register_type', 'register_addr', 'data' ]" - code => "event.set('[zeek_cols]', @zeek_ecat_registers_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ecat_registers" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ecat_registers" + init => "@zeek_ecat_registers_field_names = [ 'ts', 'orig_l2_addr', 'resp_l2_addr', 'command', 'server_addr', 'register_type', 'register_addr', 'data' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_registers_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1002,23 +957,27 @@ filter { # ecat_log_address.log # https://github.com/cisagov/icsnpp-ethercat - dissect { - id => "dissect_zeek_ecat_log_address" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][log_addr]} %{[zeek_cols][length]} %{[zeek_cols][command]} %{[zeek_cols][data]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ecat_log_address" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_ecat_log_address" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][log_addr]} %{[zeek_cols][length]} %{[zeek_cols][command]} %{[zeek_cols][data]}" + } } - ruby { - id => "ruby_zip_zeek_ecat_log_address" - init => "@zeek_ecat_log_address_field_names = [ 'ts', 'orig_l2_addr', 'resp_l2_addr', 'log_addr', 'length', 'command', 'data' ]" - code => "event.set('[zeek_cols]', @zeek_ecat_log_address_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ecat_log_address" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ecat_log_address" + init => "@zeek_ecat_log_address_field_names = [ 'ts', 'orig_l2_addr', 'resp_l2_addr', 'log_addr', 'length', 'command', 'data' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_log_address_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1035,23 +994,27 @@ filter { # ecat_dev_info.log # https://github.com/cisagov/icsnpp-ethercat - dissect { - id => "dissect_zeek_ecat_dev_info" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][server_id]} %{[zeek_cols][revision]} %{[zeek_cols][dev_type]} %{[zeek_cols][build]} %{[zeek_cols][fmmucnt]} %{[zeek_cols][smcount]} %{[zeek_cols][ports]} %{[zeek_cols][dpram]} %{[zeek_cols][features]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ecat_dev_info" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_ecat_dev_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][server_id]} %{[zeek_cols][revision]} %{[zeek_cols][dev_type]} %{[zeek_cols][build]} %{[zeek_cols][fmmucnt]} %{[zeek_cols][smcount]} %{[zeek_cols][ports]} %{[zeek_cols][dpram]} %{[zeek_cols][features]}" + } } - ruby { - id => "ruby_zip_zeek_ecat_dev_info" - init => "@zeek_ecat_dev_info_field_names = [ 'ts', 'server_id', 'revision', 'dev_type', 'build', 'fmmucnt', 'smcount', 'ports', 'dpram', 'features' ]" - code => "event.set('[zeek_cols]', @zeek_ecat_dev_info_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ecat_dev_info" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ecat_dev_info" + init => "@zeek_ecat_dev_info_field_names = [ 'ts', 'server_id', 'revision', 'dev_type', 'build', 'fmmucnt', 'smcount', 'ports', 'dpram', 'features' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_dev_info_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1068,23 +1031,27 @@ filter { # ecat_aoe_info.log # https://github.com/cisagov/icsnpp-ethercat - dissect { - id => "dissect_zeek_ecat_aoe_info" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][resp_port]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][orig_port]} %{[zeek_cols][command]} %{[zeek_cols][state]} %{[zeek_cols][data]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ecat_aoe_info" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_ecat_aoe_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][resp_port]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][orig_port]} %{[zeek_cols][command]} %{[zeek_cols][state]} %{[zeek_cols][data]}" + } } - ruby { - id => "ruby_zip_zeek_ecat_aoe_info" - init => "@zeek_ecat_aoe_info_field_names = [ 'ts', 'resp_l2_addr', 'resp_port', 'orig_l2_addr', 'orig_port', 'command', 'state', 'data' ]" - code => "event.set('[zeek_cols]', @zeek_ecat_aoe_info_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ecat_aoe_info" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ecat_aoe_info" + init => "@zeek_ecat_aoe_info_field_names = [ 'ts', 'resp_l2_addr', 'resp_port', 'orig_l2_addr', 'orig_port', 'command', 'state', 'data' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_aoe_info_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1101,23 +1068,27 @@ filter { # ecat_coe_info.log # https://github.com/cisagov/icsnpp-ethercat - dissect { - id => "dissect_zeek_ecat_coe_info" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][number]} %{[zeek_cols][type]} %{[zeek_cols][req_resp]} %{[zeek_cols][index]} %{[zeek_cols][subindex]} %{[zeek_cols][dataoffset]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ecat_coe_info" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_ecat_coe_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][number]} %{[zeek_cols][type]} %{[zeek_cols][req_resp]} %{[zeek_cols][index]} %{[zeek_cols][subindex]} %{[zeek_cols][dataoffset]}" + } } - ruby { - id => "ruby_zip_zeek_ecat_coe_info" - init => "@zeek_ecat_coe_info_field_names = [ 'ts', 'number', 'type', 'req_resp', 'index', 'subindex', 'dataoffset' ]" - code => "event.set('[zeek_cols]', @zeek_ecat_coe_info_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ecat_coe_info" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ecat_coe_info" + init => "@zeek_ecat_coe_info_field_names = [ 'ts', 'number', 'type', 'req_resp', 'index', 'subindex', 'dataoffset' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_coe_info_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1134,23 +1105,27 @@ filter { # ecat_foe_info.log # https://github.com/cisagov/icsnpp-ethercat - dissect { - id => "dissect_zeek_ecat_foe_info" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][opcode]} %{[zeek_cols][reserved]} %{[zeek_cols][packet_num]} %{[zeek_cols][error_code]} %{[zeek_cols][filename]} %{[zeek_cols][data]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ecat_foe_info" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_ecat_foe_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][opcode]} %{[zeek_cols][reserved]} %{[zeek_cols][packet_num]} %{[zeek_cols][error_code]} %{[zeek_cols][filename]} %{[zeek_cols][data]}" + } } - ruby { - id => "ruby_zip_zeek_ecat_foe_info" - init => "@zeek_ecat_foe_info_field_names = [ 'ts', 'opcode', 'reserved', 'packet_num', 'error_code', 'filename', 'data' ]" - code => "event.set('[zeek_cols]', @zeek_ecat_foe_info_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ecat_foe_info" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ecat_foe_info" + init => "@zeek_ecat_foe_info_field_names = [ 'ts', 'opcode', 'reserved', 'packet_num', 'error_code', 'filename', 'data' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_foe_info_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1167,23 +1142,27 @@ filter { # ecat_soe_info.log # https://github.com/cisagov/icsnpp-ethercat - dissect { - id => "dissect_zeek_ecat_soe_info" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][opcode]} %{[zeek_cols][incomplete]} %{[zeek_cols][error]} %{[zeek_cols][drive_num]} %{[zeek_cols][element]} %{[zeek_cols][index]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ecat_soe_info" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_ecat_soe_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][opcode]} %{[zeek_cols][incomplete]} %{[zeek_cols][error]} %{[zeek_cols][drive_num]} %{[zeek_cols][element]} %{[zeek_cols][index]}" + } } - ruby { - id => "ruby_zip_zeek_ecat_soe_info" - init => "@zeek_ecat_soe_info_field_names = [ 'ts', 'opcode', 'incomplete', 'error', 'drive_num', 'element', 'index' ]" - code => "event.set('[zeek_cols]', @zeek_ecat_soe_info_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ecat_soe_info" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ecat_soe_info" + init => "@zeek_ecat_soe_info_field_names = [ 'ts', 'opcode', 'incomplete', 'error', 'drive_num', 'element', 'index' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_soe_info_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1200,27 +1179,31 @@ filter { # ecat_arp_info.log # https://github.com/cisagov/icsnpp-ethercat # - # NOTE: I currently have this disabled by including ecat_arp_info in drop_zeek_ignored_source - # above, as it is including ALL arps and not just those from ethercat traffic which can - # be misleading (i.e., indicating ecat traffic where there is none) + # NOTE: I currently have this disabled by including ecat_arp_info in LOGSTASH_ZEEK_IGNORED_LOGS + # to be handled bdrop_zeek_ignored_source above, as it is including ALL arps and not just those + # from ethercat traffic which can be misleading (i.e., indicating ecat traffic where there is none) - dissect { - id => "dissect_zeek_ecat_arp_info" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][arp_type]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][orig_proto_addr]} %{[zeek_cols][orig_hw_addr]} %{[zeek_cols][resp_proto_addr]} %{[zeek_cols][resp_hw_addr]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ecat_arp_info" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_ecat_arp_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][arp_type]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][orig_proto_addr]} %{[zeek_cols][orig_hw_addr]} %{[zeek_cols][resp_proto_addr]} %{[zeek_cols][resp_hw_addr]}" + } } - ruby { - id => "ruby_zip_zeek_ecat_arp_info" - init => "@zeek_ecat_arp_info_field_names = [ 'ts', 'arp_type', 'orig_l2_addr', 'resp_l2_addr', 'orig_proto_addr', 'orig_hw_addr', 'resp_proto_addr', 'resp_hw_addr' ]" - code => "event.set('[zeek_cols]', @zeek_ecat_arp_info_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ecat_arp_info" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ecat_arp_info" + init => "@zeek_ecat_arp_info_field_names = [ 'ts', 'arp_type', 'orig_l2_addr', 'resp_l2_addr', 'orig_proto_addr', 'orig_hw_addr', 'resp_proto_addr', 'resp_hw_addr' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_arp_info_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1238,73 +1221,27 @@ filter { # files.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/files/main.zeek.html#type-Files::Info - if ([@metadata][zeek_fields_bitmap] and [@metadata][zeek_fields_bitmap_version]) { - - # bitmap files.log field configuration version 1 - # - # all fields (v5.1+, with extracted file size info and spicy-zip's ftime) : 0x0FFFFFFF / 268435455 - - # bitmap files.log field configuration version 0 - # all fields (< v5.1+, with extracted file size info and spicy-zip's ftime) : 0x03FFFFFF / 67108863 - - if ([@metadata][zeek_fields_bitmap_version] == 1) { - - if ([@metadata][zeek_fields_bitmap] == 268435455) { - - dissect { - id => "dissect_zeek_v51_files_with_all_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fuid]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][source]} %{[zeek_cols][depth]} %{[zeek_cols][analyzers]} %{[zeek_cols][mime_type]} %{[zeek_cols][filename]} %{[zeek_cols][duration]} %{[zeek_cols][local_orig]} %{[zeek_cols][is_orig]} %{[zeek_cols][seen_bytes]} %{[zeek_cols][total_bytes]} %{[zeek_cols][missing_bytes]} %{[zeek_cols][overflow_bytes]} %{[zeek_cols][timedout]} %{[zeek_cols][parent_fuid]} %{[zeek_cols][md5]} %{[zeek_cols][sha1]} %{[zeek_cols][sha256]} %{[zeek_cols][extracted]} %{[zeek_cols][extracted_cutoff]} %{[zeek_cols][extracted_size]} %{[zeek_cols][ftime]}" - } - } - - } else { - # who knows? the files.log preprocessed bitmap is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_files_v51_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - } else if ([@metadata][zeek_fields_bitmap_version] == 0) { - - if ([@metadata][zeek_fields_bitmap] == 67108863) { - - dissect { - id => "dissect_zeek_files_with_all_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fuid]} %{[zeek_cols][tx_hosts]} %{[zeek_cols][rx_hosts]} %{[zeek_cols][conn_uids]} %{[zeek_cols][source]} %{[zeek_cols][depth]} %{[zeek_cols][analyzers]} %{[zeek_cols][mime_type]} %{[zeek_cols][filename]} %{[zeek_cols][duration]} %{[zeek_cols][local_orig]} %{[zeek_cols][is_orig]} %{[zeek_cols][seen_bytes]} %{[zeek_cols][total_bytes]} %{[zeek_cols][missing_bytes]} %{[zeek_cols][overflow_bytes]} %{[zeek_cols][timedout]} %{[zeek_cols][parent_fuid]} %{[zeek_cols][md5]} %{[zeek_cols][sha1]} %{[zeek_cols][sha256]} %{[zeek_cols][extracted]} %{[zeek_cols][extracted_cutoff]} %{[zeek_cols][extracted_size]} %{[zeek_cols][ftime]}" - } - } - - } else { - # who knows? the files.log preprocessed bitmap is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_files_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - } else { - # who knows? the files.log preprocessed bitmap field list version is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_bitmap_files_version" - add_tag => [ "_dissectfailure" ] } - } + if "_jsonparsesuccess" in [tags] { } else { - # who knows? the files.log was not preprocessed to determine fields, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_missing_files_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_files" + dissect { + id => "dissect_zeek_v51_files_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } - } - ruby { - id => "ruby_zip_zeek_files" - init => "@zeek_files_field_names = [ 'ts', 'fuid', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'source', 'depth', 'analyzers', 'mime_type', 'filename', 'duration', 'local_orig', 'is_orig', 'seen_bytes', 'total_bytes', 'missing_bytes', 'overflow_bytes', 'timedout', 'parent_fuid', 'md5', 'sha1', 'sha256', 'extracted', 'extracted_cutoff', 'extracted_size', 'ftime' ]" - code => "event.set('[zeek_cols]', @zeek_files_field_names.zip(event.get('[message]')).to_h)" + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fuid]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][source]} %{[zeek_cols][depth]} %{[zeek_cols][analyzers]} %{[zeek_cols][mime_type]} %{[zeek_cols][filename]} %{[zeek_cols][duration]} %{[zeek_cols][local_orig]} %{[zeek_cols][is_orig]} %{[zeek_cols][seen_bytes]} %{[zeek_cols][total_bytes]} %{[zeek_cols][missing_bytes]} %{[zeek_cols][overflow_bytes]} %{[zeek_cols][timedout]} %{[zeek_cols][parent_fuid]} %{[zeek_cols][md5]} %{[zeek_cols][sha1]} %{[zeek_cols][sha256]} %{[zeek_cols][extracted]} %{[zeek_cols][extracted_cutoff]} %{[zeek_cols][extracted_size]} %{[zeek_cols][ftime]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_files" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_files" + init => "@zeek_files_field_names = [ 'ts', 'fuid', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'source', 'depth', 'analyzers', 'mime_type', 'filename', 'duration', 'local_orig', 'is_orig', 'seen_bytes', 'total_bytes', 'missing_bytes', 'overflow_bytes', 'timedout', 'parent_fuid', 'md5', 'sha1', 'sha256', 'extracted', 'extracted_cutoff', 'extracted_size', 'ftime' ]" + code => "event.set('[zeek_cols]', @zeek_files_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1350,23 +1287,27 @@ filter { # ftp.log # https://docs.zeek.org/en/stable/scripts/base/protocols/ftp/info.zeek.html#type-FTP::Info - dissect { - id => "dissect_zeek_ftp" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][command]} %{[zeek_cols][arg]} %{[zeek_cols][mime_type]} %{[zeek_cols][file_size]} %{[zeek_cols][reply_code]} %{[zeek_cols][reply_msg]} %{[zeek_cols][data_channel_passive]} %{[zeek_cols][data_channel_orig_h]} %{[zeek_cols][data_channel_resp_h]} %{[zeek_cols][data_channel_resp_p]} %{[zeek_cols][fuid]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ftp" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_ftp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][command]} %{[zeek_cols][arg]} %{[zeek_cols][mime_type]} %{[zeek_cols][file_size]} %{[zeek_cols][reply_code]} %{[zeek_cols][reply_msg]} %{[zeek_cols][data_channel_passive]} %{[zeek_cols][data_channel_orig_h]} %{[zeek_cols][data_channel_resp_h]} %{[zeek_cols][data_channel_resp_p]} %{[zeek_cols][fuid]}" + } } - ruby { - id => "ruby_zip_zeek_ftp" - init => "@zeek_ftp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'password', 'command', 'arg', 'mime_type', 'file_size', 'reply_code', 'reply_msg', 'data_channel_passive', 'data_channel_orig_h', 'data_channel_resp_h', 'data_channel_resp_p', 'fuid' ]" - code => "event.set('[zeek_cols]', @zeek_ftp_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ftp" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ftp" + init => "@zeek_ftp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'password', 'command', 'arg', 'mime_type', 'file_size', 'reply_code', 'reply_msg', 'data_channel_passive', 'data_channel_orig_h', 'data_channel_resp_h', 'data_channel_resp_p', 'fuid' ]" + code => "event.set('[zeek_cols]', @zeek_ftp_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1383,23 +1324,27 @@ filter { # genisys.log # https://github.com/cisagov/icsnpp-genisys - dissect { - id => "dissect_zeek_genisys" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][header]} %{[zeek_cols][server]} %{[zeek_cols][direction]} %{[zeek_cols][crc_transmitted]} %{[zeek_cols][crc_calculated]} %{[zeek_cols][payload_raw]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_genisys" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_genisys" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][header]} %{[zeek_cols][server]} %{[zeek_cols][direction]} %{[zeek_cols][crc_transmitted]} %{[zeek_cols][crc_calculated]} %{[zeek_cols][payload_raw]}" + } } - ruby { - id => "ruby_zip_zeek_genisys" - init => "@zeek_genisys_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'header', 'server', 'direction', 'crc_transmitted', 'crc_calculated', 'payload_raw' ]" - code => "event.set('[zeek_cols]', @zeek_genisys_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_genisys" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_genisys" + init => "@zeek_genisys_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'header', 'server', 'direction', 'crc_transmitted', 'crc_calculated', 'payload_raw' ]" + code => "event.set('[zeek_cols]', @zeek_genisys_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1416,23 +1361,27 @@ filter { # gquic.log # https://github.com/salesforce/GQUIC_Protocol_Analyzer/blob/master/scripts/Salesforce/GQUIC/main.bro - dissect { - id => "dissect_zeek_gquic" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][server_name]} %{[zeek_cols][user_agent]} %{[zeek_cols][tag_count]} %{[zeek_cols][cyu]} %{[zeek_cols][cyutags]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_gquic" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_gquic" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][server_name]} %{[zeek_cols][user_agent]} %{[zeek_cols][tag_count]} %{[zeek_cols][cyu]} %{[zeek_cols][cyutags]}" + } } - ruby { - id => "ruby_zip_zeek_gquic" - init => "@zeek_gquic_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'server_name', 'user_agent', 'tag_count', 'cyu', 'cyutags' ]" - code => "event.set('[zeek_cols]', @zeek_gquic_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_gquic" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_gquic" + init => "@zeek_gquic_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'server_name', 'user_agent', 'tag_count', 'cyu', 'cyutags' ]" + code => "event.set('[zeek_cols]', @zeek_gquic_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1449,70 +1398,27 @@ filter { # http.log # https://docs.zeek.org/en/stable/scripts/base/protocols/http/main.zeek.html#type-HTTP::Info - if ([@metadata][zeek_fields_bitmap] and [@metadata][zeek_fields_bitmap_version]) { - - # bitmap http.log field configuration version 0 - # all fields: 0x7FFFFFFFF / 34359738367 - # minus fields added by zeek-sniffpass: 0x03FFFFFFF / 1073741823 - # minus origin and fields added by zeek-sniffpass: 0x03FFFDFFF / 1073733631 - - if ([@metadata][zeek_fields_bitmap_version] == 0) { - - if ([@metadata][zeek_fields_bitmap] == 34359738367) { - dissect { - id => "dissect_zeek_http_with_all_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_depth]} %{[zeek_cols][method]} %{[zeek_cols][host]} %{[zeek_cols][uri]} %{[zeek_cols][referrer]} %{[zeek_cols][version]} %{[zeek_cols][user_agent]} %{[zeek_cols][origin]} %{[zeek_cols][request_body_len]} %{[zeek_cols][response_body_len]} %{[zeek_cols][status_code]} %{[zeek_cols][status_msg]} %{[zeek_cols][info_code]} %{[zeek_cols][info_msg]} %{[zeek_cols][tags]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][proxied]} %{[zeek_cols][orig_fuids]} %{[zeek_cols][orig_filenames]} %{[zeek_cols][orig_mime_types]} %{[zeek_cols][resp_fuids]} %{[zeek_cols][resp_filenames]} %{[zeek_cols][resp_mime_types]} %{[zeek_cols][post_username]} %{[zeek_cols][post_password_plain]} %{[zeek_cols][post_password_md5]} %{[zeek_cols][post_password_sha1]} %{[zeek_cols][post_password_sha256]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 1073741823) { - dissect { - id => "dissect_zeek_http_with_all_fields_minus_sniffpass" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_depth]} %{[zeek_cols][method]} %{[zeek_cols][host]} %{[zeek_cols][uri]} %{[zeek_cols][referrer]} %{[zeek_cols][version]} %{[zeek_cols][user_agent]} %{[zeek_cols][origin]} %{[zeek_cols][request_body_len]} %{[zeek_cols][response_body_len]} %{[zeek_cols][status_code]} %{[zeek_cols][status_msg]} %{[zeek_cols][info_code]} %{[zeek_cols][info_msg]} %{[zeek_cols][tags]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][proxied]} %{[zeek_cols][orig_fuids]} %{[zeek_cols][orig_filenames]} %{[zeek_cols][orig_mime_types]} %{[zeek_cols][resp_fuids]} %{[zeek_cols][resp_filenames]} %{[zeek_cols][resp_mime_types]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 1073733631) { - dissect { - id => "dissect_zeek_http_with_all_fields_minus_origin" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_depth]} %{[zeek_cols][method]} %{[zeek_cols][host]} %{[zeek_cols][uri]} %{[zeek_cols][referrer]} %{[zeek_cols][version]} %{[zeek_cols][user_agent]} %{[zeek_cols][request_body_len]} %{[zeek_cols][response_body_len]} %{[zeek_cols][status_code]} %{[zeek_cols][status_msg]} %{[zeek_cols][info_code]} %{[zeek_cols][info_msg]} %{[zeek_cols][tags]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][proxied]} %{[zeek_cols][orig_fuids]} %{[zeek_cols][orig_filenames]} %{[zeek_cols][orig_mime_types]} %{[zeek_cols][resp_fuids]} %{[zeek_cols][resp_filenames]} %{[zeek_cols][resp_mime_types]}" - } - } - - } else { - # who knows? the http.log preprocessed bitmap is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_http_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - } else { - # who knows? the http.log preprocessed bitmap field list version is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_bitmap_http_version" - add_tag => [ "_dissectfailure" ] } - } + if "_jsonparsesuccess" in [tags] { } else { - # who knows? the http.log was not preprocessed to determine fields, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_missing_http_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_http" + dissect { + id => "dissect_zeek_http_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_depth]} %{[zeek_cols][method]} %{[zeek_cols][host]} %{[zeek_cols][uri]} %{[zeek_cols][referrer]} %{[zeek_cols][version]} %{[zeek_cols][user_agent]} %{[zeek_cols][origin]} %{[zeek_cols][request_body_len]} %{[zeek_cols][response_body_len]} %{[zeek_cols][status_code]} %{[zeek_cols][status_msg]} %{[zeek_cols][info_code]} %{[zeek_cols][info_msg]} %{[zeek_cols][tags]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][proxied]} %{[zeek_cols][orig_fuids]} %{[zeek_cols][orig_filenames]} %{[zeek_cols][orig_mime_types]} %{[zeek_cols][resp_fuids]} %{[zeek_cols][resp_filenames]} %{[zeek_cols][resp_mime_types]} %{[zeek_cols][post_username]} %{[zeek_cols][post_password_plain]} %{[zeek_cols][post_password_md5]} %{[zeek_cols][post_password_sha1]} %{[zeek_cols][post_password_sha256]}" + } } - ruby { - id => "ruby_zip_zeek_http" - init => "@zeek_http_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'method', 'host', 'uri', 'referrer', 'version', 'user_agent', 'origin', 'request_body_len', 'response_body_len', 'status_code', 'status_msg', 'info_code', 'info_msg', 'tags', 'user', 'password', 'proxied', 'orig_fuids', 'orig_filenames', 'orig_mime_types', 'resp_fuids', 'resp_filenames', 'resp_mime_types', 'post_username', 'post_password_plain', 'post_password_md5', 'post_password_sha1', 'post_password_sha256' ]" - code => "event.set('[zeek_cols]', @zeek_http_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_http" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_http" + init => "@zeek_http_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'method', 'host', 'uri', 'referrer', 'version', 'user_agent', 'origin', 'request_body_len', 'response_body_len', 'status_code', 'status_msg', 'info_code', 'info_msg', 'tags', 'user', 'password', 'proxied', 'orig_fuids', 'orig_filenames', 'orig_mime_types', 'resp_fuids', 'resp_filenames', 'resp_mime_types', 'post_username', 'post_password_plain', 'post_password_md5', 'post_password_sha1', 'post_password_sha256' ]" + code => "event.set('[zeek_cols]', @zeek_http_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1529,23 +1435,27 @@ filter { # intel.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Info - dissect { - id => "dissect_zeek_intel" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][seen_indicator]} %{[zeek_cols][seen_indicator_type]} %{[zeek_cols][seen_where]} %{[zeek_cols][seen_node]} %{[zeek_cols][matched]} %{[zeek_cols][sources]} %{[zeek_cols][fuid]} %{[zeek_cols][file_mime_type]} %{[zeek_cols][file_desc]} %{[zeek_cols][cif_tags]} %{[zeek_cols][cif_confidence]} %{[zeek_cols][cif_source]} %{[zeek_cols][cif_description]} %{[zeek_cols][cif_firstseen]} %{[zeek_cols][cif_lastseen]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_intel" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_intel" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][seen_indicator]} %{[zeek_cols][seen_indicator_type]} %{[zeek_cols][seen_where]} %{[zeek_cols][seen_node]} %{[zeek_cols][matched]} %{[zeek_cols][sources]} %{[zeek_cols][fuid]} %{[zeek_cols][file_mime_type]} %{[zeek_cols][file_desc]} %{[zeek_cols][cif_tags]} %{[zeek_cols][cif_confidence]} %{[zeek_cols][cif_source]} %{[zeek_cols][cif_description]} %{[zeek_cols][cif_firstseen]} %{[zeek_cols][cif_lastseen]}" + } } - ruby { - id => "ruby_zip_zeek_intel" - init => "@zeek_intel_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'seen_indicator', 'seen_indicator_type', 'seen_where', 'seen_node', 'matched', 'sources', 'fuid', 'file_mime_type', 'file_desc', 'cif_tags', 'cif_confidence', 'cif_source', 'cif_description', 'cif_firstseen', 'cif_lastseen' ]" - code => "event.set('[zeek_cols]', @zeek_intel_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_intel" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_intel" + init => "@zeek_intel_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'seen_indicator', 'seen_indicator_type', 'seen_where', 'seen_node', 'matched', 'sources', 'fuid', 'file_mime_type', 'file_desc', 'cif_tags', 'cif_confidence', 'cif_source', 'cif_description', 'cif_firstseen', 'cif_lastseen' ]" + code => "event.set('[zeek_cols]', @zeek_intel_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1554,23 +1464,27 @@ filter { # ipsec.log # https://github.com/corelight/zeek-spicy-ipsec/blob/master/analyzer/main.zeek - dissect { - id => "dissect_zeek_ipsec" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][initiator_spi]} %{[zeek_cols][responder_spi]} %{[zeek_cols][maj_ver]} %{[zeek_cols][min_ver]} %{[zeek_cols][exchange_type]} %{[zeek_cols][flag_e]} %{[zeek_cols][flag_c]} %{[zeek_cols][flag_a]} %{[zeek_cols][flag_i]} %{[zeek_cols][flag_v]} %{[zeek_cols][flag_r]} %{[zeek_cols][message_id]} %{[zeek_cols][vendor_ids]} %{[zeek_cols][notify_messages]} %{[zeek_cols][transforms]} %{[zeek_cols][ke_dh_groups]} %{[zeek_cols][proposals]} %{[zeek_cols][protocol_id]} %{[zeek_cols][certificates]} %{[zeek_cols][transform_attributes]} %{[zeek_cols][length]} %{[zeek_cols][hash]} %{[zeek_cols][doi]} %{[zeek_cols][situation]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ipsec" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_ipsec" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][initiator_spi]} %{[zeek_cols][responder_spi]} %{[zeek_cols][maj_ver]} %{[zeek_cols][min_ver]} %{[zeek_cols][exchange_type]} %{[zeek_cols][flag_e]} %{[zeek_cols][flag_c]} %{[zeek_cols][flag_a]} %{[zeek_cols][flag_i]} %{[zeek_cols][flag_v]} %{[zeek_cols][flag_r]} %{[zeek_cols][message_id]} %{[zeek_cols][vendor_ids]} %{[zeek_cols][notify_messages]} %{[zeek_cols][transforms]} %{[zeek_cols][ke_dh_groups]} %{[zeek_cols][proposals]} %{[zeek_cols][protocol_id]} %{[zeek_cols][certificates]} %{[zeek_cols][transform_attributes]} %{[zeek_cols][length]} %{[zeek_cols][hash]} %{[zeek_cols][doi]} %{[zeek_cols][situation]}" + } } - ruby { - id => "ruby_zip_zeek_ipsec" - init => "@zeek_ipsec_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'is_orig', 'initiator_spi', 'responder_spi', 'maj_ver', 'min_ver', 'exchange_type', 'flag_e', 'flag_c', 'flag_a', 'flag_i', 'flag_v', 'flag_r', 'message_id', 'vendor_ids', 'notify_messages', 'transforms', 'ke_dh_groups', 'proposals', 'protocol_id', 'certificates', 'transform_attributes', 'length', 'hash', 'doi', 'situation' ]" - code => "event.set('[zeek_cols]', @zeek_ipsec_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ipsec" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ipsec" + init => "@zeek_ipsec_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'is_orig', 'initiator_spi', 'responder_spi', 'maj_ver', 'min_ver', 'exchange_type', 'flag_e', 'flag_c', 'flag_a', 'flag_i', 'flag_v', 'flag_r', 'message_id', 'vendor_ids', 'notify_messages', 'transforms', 'ke_dh_groups', 'proposals', 'protocol_id', 'certificates', 'transform_attributes', 'length', 'hash', 'doi', 'situation' ]" + code => "event.set('[zeek_cols]', @zeek_ipsec_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1586,23 +1500,27 @@ filter { # irc.log # https://docs.zeek.org/en/stable/scripts/base/protocols/irc/main.zeek.html#type-IRC::Info - dissect { - id => "dissect_zeek_irc" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][nick]} %{[zeek_cols][user]} %{[zeek_cols][command]} %{[zeek_cols][value]} %{[zeek_cols][addl]} %{[zeek_cols][dcc_file_name]} %{[zeek_cols][dcc_file_size]} %{[zeek_cols][dcc_mime_type]} %{[zeek_cols][fuid]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_irc" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_irc" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][nick]} %{[zeek_cols][user]} %{[zeek_cols][command]} %{[zeek_cols][value]} %{[zeek_cols][addl]} %{[zeek_cols][dcc_file_name]} %{[zeek_cols][dcc_file_size]} %{[zeek_cols][dcc_mime_type]} %{[zeek_cols][fuid]}" + } } - ruby { - id => "ruby_zip_zeek_irc" - init => "@zeek_irc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'nick', 'user', 'command', 'value', 'addl', 'dcc_file_name', 'dcc_file_size', 'dcc_mime_type', 'fuid' ]" - code => "event.set('[zeek_cols]', @zeek_irc_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_irc" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_irc" + init => "@zeek_irc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'nick', 'user', 'command', 'value', 'addl', 'dcc_file_name', 'dcc_file_size', 'dcc_mime_type', 'fuid' ]" + code => "event.set('[zeek_cols]', @zeek_irc_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1619,23 +1537,27 @@ filter { # cotp.log # https://github.com/cisagov/icsnpp-s7comm - dissect { - id => "dissect_zeek_cotp" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][pdu_code]} %{[zeek_cols][pdu_name]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_cotp" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_cotp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][pdu_code]} %{[zeek_cols][pdu_name]}" + } } - ruby { - id => "ruby_zip_zeek_cotp" - init => "@zeek_cotp_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'pdu_code', 'pdu_name' ]" - code => "event.set('[zeek_cols]', @zeek_cotp_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_cotp" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_cotp" + init => "@zeek_cotp_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'pdu_code', 'pdu_name' ]" + code => "event.set('[zeek_cols]', @zeek_cotp_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1653,23 +1575,27 @@ filter { # kerberos.log # https://docs.zeek.org/en/stable/scripts/base/protocols/krb/main.zeek.html#type-KRB::Info - dissect { - id => "dissect_zeek_kerberos" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][request_type]} %{[zeek_cols][cname]} %{[zeek_cols][sname]} %{[zeek_cols][success]} %{[zeek_cols][error_msg]} %{[zeek_cols][from]} %{[zeek_cols][till]} %{[zeek_cols][cipher]} %{[zeek_cols][forwardable]} %{[zeek_cols][renewable]} %{[zeek_cols][client_cert_subject]} %{[zeek_cols][client_cert_fuid]} %{[zeek_cols][server_cert_subject]} %{[zeek_cols][server_cert_fuid]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_kerberos" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_kerberos" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][request_type]} %{[zeek_cols][cname]} %{[zeek_cols][sname]} %{[zeek_cols][success]} %{[zeek_cols][error_msg]} %{[zeek_cols][from]} %{[zeek_cols][till]} %{[zeek_cols][cipher]} %{[zeek_cols][forwardable]} %{[zeek_cols][renewable]} %{[zeek_cols][client_cert_subject]} %{[zeek_cols][client_cert_fuid]} %{[zeek_cols][server_cert_subject]} %{[zeek_cols][server_cert_fuid]}" + } } - ruby { - id => "ruby_zip_zeek_kerberos" - init => "@zeek_kerberos_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'request_type', 'cname', 'sname', 'success', 'error_msg', 'from', 'till', 'cipher', 'forwardable', 'renewable', 'client_cert_subject', 'client_cert_fuid', 'server_cert_subject', 'server_cert_fuid' ]" - code => "event.set('[zeek_cols]', @zeek_kerberos_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_kerberos" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_kerberos" + init => "@zeek_kerberos_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'request_type', 'cname', 'sname', 'success', 'error_msg', 'from', 'till', 'cipher', 'forwardable', 'renewable', 'client_cert_subject', 'client_cert_fuid', 'server_cert_subject', 'server_cert_fuid' ]" + code => "event.set('[zeek_cols]', @zeek_kerberos_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1683,23 +1609,27 @@ filter { # known_certs.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/ssl/known-certs.zeek.html#type-Known::CertsInfo - dissect { - id => "dissect_zeek_known_certs" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][subject]} %{[zeek_cols][issuer_subject]} %{[zeek_cols][serial]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_known_certs" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_known_certs" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][subject]} %{[zeek_cols][issuer_subject]} %{[zeek_cols][serial]}" + } } - ruby { - id => "ruby_zip_zeek_known_certs" - init => "@zeek_known_certs_field_names = [ 'ts', 'orig_h', 'orig_p', 'subject', 'resp_h', 'issuer_subject', 'serial' ]" - code => "event.set('[zeek_cols]', @zeek_known_certs_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_known_certs" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_known_certs" + init => "@zeek_known_certs_field_names = [ 'ts', 'orig_h', 'orig_p', 'subject', 'resp_h', 'issuer_subject', 'serial' ]" + code => "event.set('[zeek_cols]', @zeek_known_certs_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1716,23 +1646,27 @@ filter { # known_hosts.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/conn/known-hosts.zeek.html#type-Known::HostsInfo - dissect { - id => "dissect_zeek_known_hosts" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_known_hosts" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_known_hosts" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]}" + } } - ruby { - id => "ruby_zip_zeek_known_hosts" - init => "@zeek_known_hosts_field_names = [ 'ts', 'orig_h' ]" - code => "event.set('[zeek_cols]', @zeek_known_hosts_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_known_hosts" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_known_hosts" + init => "@zeek_known_hosts_field_names = [ 'ts', 'orig_h' ]" + code => "event.set('[zeek_cols]', @zeek_known_hosts_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1741,23 +1675,27 @@ filter { # known_modbus.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/modbus/known-masters-slaves.zeek.html#type-Known::ModbusInfo - dissect { - id => "dissect_zeek_known_modbus" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]} %{[zeek_cols][device_type]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_known_modbus" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_known_modbus" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]} %{[zeek_cols][device_type]}" + } } - ruby { - id => "ruby_zip_zeek_known_modbus" - init => "@zeek_known_modbus_field_names = [ 'ts', 'orig_h', 'device_type' ]" - code => "event.set('[zeek_cols]', @zeek_known_modbus_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_known_modbus" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_known_modbus" + init => "@zeek_known_modbus_field_names = [ 'ts', 'orig_h', 'device_type' ]" + code => "event.set('[zeek_cols]', @zeek_known_modbus_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1778,23 +1716,27 @@ filter { # known_services.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/conn/known-services.zeek.html#type-Known::ServicesInfo - dissect { - id => "dissect_zeek_known_services" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_known_services" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_known_services" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]}" + } } - ruby { - id => "ruby_zip_zeek_known_services" - init => "@zeek_known_services_field_names = [ 'ts', 'resp_h', 'resp_p', 'proto', 'service' ]" - code => "event.set('[zeek_cols]', @zeek_known_services_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_known_services" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_known_services" + init => "@zeek_known_services_field_names = [ 'ts', 'resp_h', 'resp_p', 'proto', 'service' ]" + code => "event.set('[zeek_cols]', @zeek_known_services_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1824,24 +1766,27 @@ filter { # ldap.log # main.zeek (https://docs.zeek.org/en/master/scripts/base/protocols/ldap/main.zeek.html) - dissect { - id => "dissect_zeek_ldap" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][message_id]} %{[zeek_cols][version]} %{[zeek_cols][operation]} %{[zeek_cols][result_code]} %{[zeek_cols][result_message]} %{[zeek_cols][object]} %{[zeek_cols][argument]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ldap" + } else { + dissect { + id => "dissect_zeek_ldap" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][message_id]} %{[zeek_cols][version]} %{[zeek_cols][operation]} %{[zeek_cols][result_code]} %{[zeek_cols][result_message]} %{[zeek_cols][object]} %{[zeek_cols][argument]}" + } } - ruby { - id => "ruby_zip_zeek_ldap" - init => "@zeek_ldap_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'message_id', 'version', 'operation', 'result_code', 'result_message', 'object', 'argument' ]" - code => "event.set('[zeek_cols]', @zeek_ldap_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ldap" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ldap" + init => "@zeek_ldap_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'message_id', 'version', 'operation', 'result_code', 'result_message', 'object', 'argument' ]" + code => "event.set('[zeek_cols]', @zeek_ldap_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1858,24 +1803,27 @@ filter { # ldap_search.log # main.zeek (https://docs.zeek.org/en/master/scripts/base/protocols/ldap/main.zeek.html) - dissect { - id => "dissect_zeek_ldap_search" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][message_id]} %{[zeek_cols][scope]} %{[zeek_cols][deref]} %{[zeek_cols][base_object]} %{[zeek_cols][result_count]} %{[zeek_cols][result_code]} %{[zeek_cols][result_message]} %{[zeek_cols][filter]} %{[zeek_cols][attributes]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ldap_search" + } else { + dissect { + id => "dissect_zeek_ldap_search" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][message_id]} %{[zeek_cols][scope]} %{[zeek_cols][deref]} %{[zeek_cols][base_object]} %{[zeek_cols][result_count]} %{[zeek_cols][result_code]} %{[zeek_cols][result_message]} %{[zeek_cols][filter]} %{[zeek_cols][attributes]}" + } } - ruby { - id => "ruby_zip_zeek_ldap_search" - init => "@zeek_ldap_search_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'message_id', 'scope', 'deref', 'base_object', 'result_count', 'result_code', 'result_message', 'filter', 'attributes' ]" - code => "event.set('[zeek_cols]', @zeek_ldap_search_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ldap_search" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ldap_search" + init => "@zeek_ldap_search_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'message_id', 'scope', 'deref', 'base_object', 'result_count', 'result_code', 'result_message', 'filter', 'attributes' ]" + code => "event.set('[zeek_cols]', @zeek_ldap_search_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1892,23 +1840,27 @@ filter { # login.log # custom login.log module (rudimentary, telnet/rlogin/rsh analyzers are old and not the greatest) - dissect { - id => "dissect_zeek_login" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][service]} %{[zeek_cols][success]} %{[zeek_cols][confused]} %{[zeek_cols][user]} %{[zeek_cols][client_user]} %{[zeek_cols][password]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_login" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_login" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } - } - ruby { - id => "ruby_zip_zeek_login" - init => "@zeek_login_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'service', 'success', 'confused', 'user', 'client_user', 'password' ]" - code => "event.set('[zeek_cols]', @zeek_login_field_names.zip(event.get('[message]')).to_h)" + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][service]} %{[zeek_cols][success]} %{[zeek_cols][confused]} %{[zeek_cols][user]} %{[zeek_cols][client_user]} %{[zeek_cols][password]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_login" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_login" + init => "@zeek_login_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'service', 'success', 'confused', 'user', 'client_user', 'password' ]" + code => "event.set('[zeek_cols]', @zeek_login_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1917,23 +1869,27 @@ filter { # modbus.log # https://docs.zeek.org/en/stable/scripts/base/protocols/modbus/main.zeek.html#type-Modbus::Info - dissect { - id => "dissect_zeek_modbus" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_id]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][exception]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_modbus" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_modbus" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_id]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][exception]}" + } } - ruby { - id => "ruby_zip_zeek_modbus" - init => "@zeek_modbus_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'exception' ]" - code => "event.set('[zeek_cols]', @zeek_modbus_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_modbus" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_modbus" + init => "@zeek_modbus_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'exception' ]" + code => "event.set('[zeek_cols]', @zeek_modbus_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1948,24 +1904,27 @@ filter { # modbus_detailed.log # main.zeek (https://github.com/cisagov/icsnpp-modbus) - dissect { - id => "dissect_zeek_modbus_detailed" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_id]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][address]} %{[zeek_cols][quantity]} %{[zeek_cols][values]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_modbus_detailed" + } else { + dissect { + id => "dissect_zeek_modbus_detailed" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_id]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][address]} %{[zeek_cols][quantity]} %{[zeek_cols][values]}" + } } - ruby { - id => "ruby_zip_zeek_modbus_detailed" - init => "@zeek_modbus_detailed_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'address', 'quantity', 'values' ]" - code => "event.set('[zeek_cols]', @zeek_modbus_detailed_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_modbus_detailed" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_modbus_detailed" + init => "@zeek_modbus_detailed_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'address', 'quantity', 'values' ]" + code => "event.set('[zeek_cols]', @zeek_modbus_detailed_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1982,24 +1941,27 @@ filter { # modbus_mask_write_register.log # main.zeek (https://github.com/cisagov/icsnpp-modbus) - dissect { - id => "dissect_zeek_modbus_mask_write_register" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_id]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][address]} %{[zeek_cols][and_mask]} %{[zeek_cols][or_mask]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_modbus_mask_write_register" + } else { + dissect { + id => "dissect_zeek_modbus_mask_write_register" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_id]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][address]} %{[zeek_cols][and_mask]} %{[zeek_cols][or_mask]}" + } } - ruby { - id => "ruby_zip_zeek_modbus_mask_write_register" - init => "@zeek_modbus_mask_write_register_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'address', 'and_mask', 'or_mask' ]" - code => "event.set('[zeek_cols]', @zeek_modbus_mask_write_register_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_modbus_mask_write_register" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_modbus_mask_write_register" + init => "@zeek_modbus_mask_write_register_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'address', 'and_mask', 'or_mask' ]" + code => "event.set('[zeek_cols]', @zeek_modbus_mask_write_register_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2016,24 +1978,27 @@ filter { # modbus_read_device_identification.log # main.zeek (https://github.com/cisagov/icsnpp-modbus) - dissect { - id => "dissect_zeek_modbus_read_device_identification" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_id]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][mei_type]} %{[zeek_cols][conformity_level_code]} %{[zeek_cols][conformity_level]} %{[zeek_cols][device_id_code]} %{[zeek_cols][object_id_code]} %{[zeek_cols][object_id]} %{[zeek_cols][object_value]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_modbus_read_device_identification" + } else { + dissect { + id => "dissect_zeek_modbus_read_device_identification" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_id]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][mei_type]} %{[zeek_cols][conformity_level_code]} %{[zeek_cols][conformity_level]} %{[zeek_cols][device_id_code]} %{[zeek_cols][object_id_code]} %{[zeek_cols][object_id]} %{[zeek_cols][object_value]}" + } } - ruby { - id => "ruby_zip_zeek_modbus_read_device_identification" - init => "@zeek_modbus_read_device_identification_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'mei_type', 'conformity_level_code', 'conformity_level', 'device_id_code', 'object_id_code', 'object_id', 'object_value' ]" - code => "event.set('[zeek_cols]', @zeek_modbus_read_device_identification_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_modbus_read_device_identification" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_modbus_read_device_identification" + init => "@zeek_modbus_read_device_identification_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'mei_type', 'conformity_level_code', 'conformity_level', 'device_id_code', 'object_id_code', 'object_id', 'object_value' ]" + code => "event.set('[zeek_cols]', @zeek_modbus_read_device_identification_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2050,24 +2015,27 @@ filter { # modbus_read_write_multiple_registers.log # main.zeek (https://github.com/cisagov/icsnpp-modbus) - dissect { - id => "dissect_zeek_modbus_read_write_multiple_registers" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][`]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][write_start_address]} %{[zeek_cols][write_registers]} %{[zeek_cols][read_start_address]} %{[zeek_cols][read_quantity]} %{[zeek_cols][read_registers]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_modbus_read_write_multiple_registers" + } else { + dissect { + id => "dissect_zeek_modbus_read_write_multiple_registers" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][`]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][write_start_address]} %{[zeek_cols][write_registers]} %{[zeek_cols][read_start_address]} %{[zeek_cols][read_quantity]} %{[zeek_cols][read_registers]}" + } } - ruby { - id => "ruby_zip_zeek_modbus_read_write_multiple_registers" - init => "@zeek_modbus_read_write_multiple_registers_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'write_start_address', 'write_registers', 'read_start_address', 'read_quantity', 'read_registers' ]" - code => "event.set('[zeek_cols]', @zeek_modbus_read_write_multiple_registers_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_modbus_read_write_multiple_registers" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_modbus_read_write_multiple_registers" + init => "@zeek_modbus_read_write_multiple_registers_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'write_start_address', 'write_registers', 'read_start_address', 'read_quantity', 'read_registers' ]" + code => "event.set('[zeek_cols]', @zeek_modbus_read_write_multiple_registers_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2084,23 +2052,27 @@ filter { # mqtt_connect.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/mqtt/main.zeek.html#type-MQTT::ConnectInfo - dissect { - id => "dissect_zeek_mqtt_connect" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto_name]} %{[zeek_cols][proto_version]} %{[zeek_cols][client_id]} %{[zeek_cols][connect_status]} %{[zeek_cols][will_topic]} %{[zeek_cols][will_payload]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_mqtt_connect" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_mqtt_connect" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto_name]} %{[zeek_cols][proto_version]} %{[zeek_cols][client_id]} %{[zeek_cols][connect_status]} %{[zeek_cols][will_topic]} %{[zeek_cols][will_payload]}" + } } - ruby { - id => "ruby_zip_zeek_mqtt_connect" - init => "@zeek_mqtt_connect_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto_name', 'proto_version', 'client_id', 'connect_status', 'will_topic', 'will_payload' ]" - code => "event.set('[zeek_cols]', @zeek_mqtt_connect_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_mqtt_connect" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_mqtt_connect" + init => "@zeek_mqtt_connect_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto_name', 'proto_version', 'client_id', 'connect_status', 'will_topic', 'will_payload' ]" + code => "event.set('[zeek_cols]', @zeek_mqtt_connect_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2117,23 +2089,27 @@ filter { # mqtt_publish.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/mqtt/main.zeek.html#type-MQTT::PublishInfo - dissect { - id => "dissect_zeek_mqtt_publish" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][from_client]} %{[zeek_cols][retain]} %{[zeek_cols][qos]} %{[zeek_cols][status]} %{[zeek_cols][topic]} %{[zeek_cols][payload]} %{[zeek_cols][payload_len]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_mqtt_publish" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_mqtt_publish" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][from_client]} %{[zeek_cols][retain]} %{[zeek_cols][qos]} %{[zeek_cols][status]} %{[zeek_cols][topic]} %{[zeek_cols][payload]} %{[zeek_cols][payload_len]}" + } } - ruby { - id => "ruby_zip_zeek_mqtt_publish" - init => "@zeek_mqtt_publish_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'from_client', 'retain', 'qos', 'status', 'topic', 'payload', 'payload_len' ]" - code => "event.set('[zeek_cols]', @zeek_mqtt_publish_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_mqtt_publish" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_mqtt_publish" + init => "@zeek_mqtt_publish_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'from_client', 'retain', 'qos', 'status', 'topic', 'payload', 'payload_len' ]" + code => "event.set('[zeek_cols]', @zeek_mqtt_publish_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2150,23 +2126,27 @@ filter { # mqtt_subscribe.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/mqtt/main.zeek.html#type-MQTT::SubscribeInfo - dissect { - id => "dissect_zeek_mqtt_subscribe" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][action]} %{[zeek_cols][topics]} %{[zeek_cols][qos_levels]} %{[zeek_cols][granted_qos_level]} %{[zeek_cols][ack]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_mqtt_subscribe" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_mqtt_subscribe" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][action]} %{[zeek_cols][topics]} %{[zeek_cols][qos_levels]} %{[zeek_cols][granted_qos_level]} %{[zeek_cols][ack]}" + } } - ruby { - id => "ruby_zip_zeek_mqtt_subscribe" - init => "@zeek_mqtt_subscribe_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'action', 'topics', 'qos_levels', 'granted_qos_level', 'ack' ]" - code => "event.set('[zeek_cols]', @zeek_mqtt_subscribe_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_mqtt_subscribe" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_mqtt_subscribe" + init => "@zeek_mqtt_subscribe_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'action', 'topics', 'qos_levels', 'granted_qos_level', 'ack' ]" + code => "event.set('[zeek_cols]', @zeek_mqtt_subscribe_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2186,23 +2166,27 @@ filter { # mysql.log # https://docs.zeek.org/en/stable/scripts/base/protocols/mysql/main.zeek.html#type-MySQL::Info - dissect { - id => "dissect_zeek_mysql" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][cmd]} %{[zeek_cols][arg]} %{[zeek_cols][success]} %{[zeek_cols][rows]} %{[zeek_cols][response]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_mysql" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_mysql" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][cmd]} %{[zeek_cols][arg]} %{[zeek_cols][success]} %{[zeek_cols][rows]} %{[zeek_cols][response]}" + } } - ruby { - id => "ruby_zip_zeek_mysql" - init => "@zeek_mysql_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'cmd', 'arg', 'success', 'rows', 'response' ]" - code => "event.set('[zeek_cols]', @zeek_mysql_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_mysql" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_mysql" + init => "@zeek_mysql_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'cmd', 'arg', 'success', 'rows', 'response' ]" + code => "event.set('[zeek_cols]', @zeek_mysql_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2216,24 +2200,27 @@ filter { # notice.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/notice/main.zeek.html#type-Notice::Info - dissect { - id => "dissect_zeek_notice_with_all_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][fuid]} %{[zeek_cols][file_mime_type]} %{[zeek_cols][file_desc]} %{[zeek_cols][proto]} %{[zeek_cols][note]} %{[zeek_cols][msg]} %{[zeek_cols][sub]} %{[zeek_cols][src]} %{[zeek_cols][dst]} %{[zeek_cols][p]} %{[zeek_cols][n]} %{[zeek_cols][peer_descr]} %{[zeek_cols][actions]} %{[zeek_cols][email_dest]} %{[zeek_cols][suppress_for]} %{[zeek_cols][remote_location_country_code]} %{[zeek_cols][remote_location_region]} %{[zeek_cols][remote_location_city]} %{[zeek_cols][remote_location_latitude]} %{[zeek_cols][remote_location_longitude]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_notice" + } else { + dissect { + id => "dissect_zeek_notice_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][fuid]} %{[zeek_cols][file_mime_type]} %{[zeek_cols][file_desc]} %{[zeek_cols][proto]} %{[zeek_cols][note]} %{[zeek_cols][msg]} %{[zeek_cols][sub]} %{[zeek_cols][src]} %{[zeek_cols][dst]} %{[zeek_cols][p]} %{[zeek_cols][n]} %{[zeek_cols][peer_descr]} %{[zeek_cols][actions]} %{[zeek_cols][email_dest]} %{[zeek_cols][suppress_for]} %{[zeek_cols][remote_location_country_code]} %{[zeek_cols][remote_location_region]} %{[zeek_cols][remote_location_city]} %{[zeek_cols][remote_location_latitude]} %{[zeek_cols][remote_location_longitude]}" + } } - ruby { - id => "ruby_zip_zeek_notice" - init => "@zeek_notice_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'fuid', 'file_mime_type', 'file_desc', 'proto', 'note', 'msg', 'sub', 'src', 'dst', 'p', 'n', 'peer_descr', 'actions', 'email_dest', 'suppress_for', 'remote_location_country_code', 'remote_location_region', 'remote_location_city', 'remote_location_latitude', 'remote_location_longitude' ]" - code => "event.set('[zeek_cols]', @zeek_notice_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_notice" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_notice" + init => "@zeek_notice_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'fuid', 'file_mime_type', 'file_desc', 'proto', 'note', 'msg', 'sub', 'src', 'dst', 'p', 'n', 'peer_descr', 'actions', 'email_dest', 'suppress_for', 'remote_location_country_code', 'remote_location_region', 'remote_location_city', 'remote_location_latitude', 'remote_location_longitude' ]" + code => "event.set('[zeek_cols]', @zeek_notice_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2258,24 +2245,27 @@ filter { # ntlm.log # https://docs.zeek.org/en/stable/scripts/base/protocols/ntlm/main.zeek.html#type-NTLM::Info - dissect { - id => "dissect_zeek_ntlm_with_all_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user]} %{[zeek_cols][host]} %{[zeek_cols][domain]} %{[zeek_cols][server_nb_computer]} %{[zeek_cols][server_dns_computer]} %{[zeek_cols][server_tree]} %{[zeek_cols][success]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ntlm" + } else { + dissect { + id => "dissect_zeek_ntlm_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user]} %{[zeek_cols][host]} %{[zeek_cols][domain]} %{[zeek_cols][server_nb_computer]} %{[zeek_cols][server_dns_computer]} %{[zeek_cols][server_tree]} %{[zeek_cols][success]}" + } } - ruby { - id => "ruby_zip_zeek_ntlm" - init => "@zeek_ntlm_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'host', 'domain', 'server_nb_computer', 'server_dns_computer', 'server_tree', 'success' ]" - code => "event.set('[zeek_cols]', @zeek_ntlm_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ntlm" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ntlm" + init => "@zeek_ntlm_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'host', 'domain', 'server_nb_computer', 'server_dns_computer', 'server_tree', 'success' ]" + code => "event.set('[zeek_cols]', @zeek_ntlm_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2289,23 +2279,27 @@ filter { # ntp.log # https://docs.zeek.org/en/latest/scripts/base/protocols/ntp/main.zeek.html#type-NTP::Info - dissect { - id => "dissect_zeek_ntp" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][mode]} %{[zeek_cols][stratum]} %{[zeek_cols][poll]} %{[zeek_cols][precision]} %{[zeek_cols][root_delay]} %{[zeek_cols][root_disp]} %{[zeek_cols][ref_id]} %{[zeek_cols][ref_time]} %{[zeek_cols][org_time]} %{[zeek_cols][rec_time]} %{[zeek_cols][xmt_time]} %{[zeek_cols][num_exts]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ntp" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_ntp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][mode]} %{[zeek_cols][stratum]} %{[zeek_cols][poll]} %{[zeek_cols][precision]} %{[zeek_cols][root_delay]} %{[zeek_cols][root_disp]} %{[zeek_cols][ref_id]} %{[zeek_cols][ref_time]} %{[zeek_cols][org_time]} %{[zeek_cols][rec_time]} %{[zeek_cols][xmt_time]} %{[zeek_cols][num_exts]}" + } } - ruby { - id => "ruby_zip_zeek_ntp" - init => "@zeek_ntp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'mode', 'stratum', 'poll', 'precision', 'root_delay', 'root_disp', 'ref_id', 'ref_time', 'org_time', 'rec_time', 'xmt_time', 'num_exts' ]" - code => "event.set('[zeek_cols]', @zeek_ntp_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ntp" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ntp" + init => "@zeek_ntp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'mode', 'stratum', 'poll', 'precision', 'root_delay', 'root_disp', 'ref_id', 'ref_time', 'org_time', 'rec_time', 'xmt_time', 'num_exts' ]" + code => "event.set('[zeek_cols]', @zeek_ntp_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2322,24 +2316,28 @@ filter { # ocsp.log # https://docs.zeek.org/en/stable/scripts/policy/files/x509/log-ocsp.zeek.html#type-OCSP::Info - dissect { - id => "dissect_zeek_ocsp" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fuid]} %{[zeek_cols][hashAlgorithm]} %{[zeek_cols][issuerNameHash]} %{[zeek_cols][issuerKeyHash]} %{[zeek_cols][serialNumber]} %{[zeek_cols][certStatus]} %{[zeek_cols][revoketime]} %{[zeek_cols][revokereason]} %{[zeek_cols][thisUpdate]} %{[zeek_cols][nextUpdate]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ocsp" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_ocsp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fuid]} %{[zeek_cols][hashAlgorithm]} %{[zeek_cols][issuerNameHash]} %{[zeek_cols][issuerKeyHash]} %{[zeek_cols][serialNumber]} %{[zeek_cols][certStatus]} %{[zeek_cols][revoketime]} %{[zeek_cols][revokereason]} %{[zeek_cols][thisUpdate]} %{[zeek_cols][nextUpdate]}" + } } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ocsp" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } - ruby { - id => "ruby_zip_zeek_ocsp" - init => "@zeek_ocsp_field_names = [ 'ts', 'fuid', 'hashAlgorithm', 'issuerNameHash', 'issuerKeyHash', 'serialNumber', 'certStatus', 'revoketime', 'revokereason', 'thisUpdate', 'nextUpdate' ]" - code => "event.set('[zeek_cols]', @zeek_ocsp_field_names.zip(event.get('[message]')).to_h)" + ruby { + id => "ruby_zip_zeek_ocsp" + init => "@zeek_ocsp_field_names = [ 'ts', 'fuid', 'hashAlgorithm', 'issuerNameHash', 'issuerKeyHash', 'serialNumber', 'certStatus', 'revoketime', 'revokereason', 'thisUpdate', 'nextUpdate' ]" + code => "event.set('[zeek_cols]', @zeek_ocsp_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2351,24 +2349,28 @@ filter { # ospf.log # https://github.com/corelight/zeek-spicy-ospf - dissect { - id => "dissect_zeek_ospf" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]} %{[zeek_cols][resp_h]} %{[zeek_cols][ospf_type]} %{[zeek_cols][version]} %{[zeek_cols][router_id]} %{[zeek_cols][area_id]} %{[zeek_cols][interface_id]} %{[zeek_cols][netmask]} %{[zeek_cols][desig_router]} %{[zeek_cols][backup_router]} %{[zeek_cols][neighbors]} %{[zeek_cols][lsa_type]} %{[zeek_cols][link_state_id]} %{[zeek_cols][advert_router]} %{[zeek_cols][routers]} %{[zeek_cols][link_id]} %{[zeek_cols][link_data]} %{[zeek_cols][link_type]} %{[zeek_cols][neighbor_router_id]} %{[zeek_cols][metrics]} %{[zeek_cols][fwd_addrs]} %{[zeek_cols][route_tags]} %{[zeek_cols][neighbor_interface_id]} %{[zeek_cols][prefix]} %{[zeek_cols][metric]} %{[zeek_cols][dest_router_id]} %{[zeek_cols][link_prefixes]} %{[zeek_cols][intra_prefixes]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ospf" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_ospf" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]} %{[zeek_cols][resp_h]} %{[zeek_cols][ospf_type]} %{[zeek_cols][version]} %{[zeek_cols][router_id]} %{[zeek_cols][area_id]} %{[zeek_cols][interface_id]} %{[zeek_cols][netmask]} %{[zeek_cols][desig_router]} %{[zeek_cols][backup_router]} %{[zeek_cols][neighbors]} %{[zeek_cols][lsa_type]} %{[zeek_cols][link_state_id]} %{[zeek_cols][advert_router]} %{[zeek_cols][routers]} %{[zeek_cols][link_id]} %{[zeek_cols][link_data]} %{[zeek_cols][link_type]} %{[zeek_cols][neighbor_router_id]} %{[zeek_cols][metrics]} %{[zeek_cols][fwd_addrs]} %{[zeek_cols][route_tags]} %{[zeek_cols][neighbor_interface_id]} %{[zeek_cols][prefix]} %{[zeek_cols][metric]} %{[zeek_cols][dest_router_id]} %{[zeek_cols][link_prefixes]} %{[zeek_cols][intra_prefixes]}" + } } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ospf" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } - ruby { - id => "ruby_zip_zeek_ospf" - init => "@zeek_ospf_field_names = [ 'ts', 'orig_h', 'resp_h', 'ospf_type', 'version', 'router_id', 'area_id', 'interface_id', 'netmask', 'desig_router', 'backup_router', 'neighbors', 'lsa_type', 'link_state_id', 'advert_router', 'routers', 'link_id', 'link_data', 'link_type', 'neighbor_router_id', 'metrics', 'fwd_addrs', 'route_tags', 'neighbor_interface_id', 'prefix', 'metric', 'dest_router_id', 'link_prefixes', 'intra_prefixes' ]" - code => "event.set('[zeek_cols]', @zeek_ospf_field_names.zip(event.get('[message]')).to_h)" + ruby { + id => "ruby_zip_zeek_ospf" + init => "@zeek_ospf_field_names = [ 'ts', 'orig_h', 'resp_h', 'ospf_type', 'version', 'router_id', 'area_id', 'interface_id', 'netmask', 'desig_router', 'backup_router', 'neighbors', 'lsa_type', 'link_state_id', 'advert_router', 'routers', 'link_id', 'link_data', 'link_type', 'neighbor_router_id', 'metrics', 'fwd_addrs', 'route_tags', 'neighbor_interface_id', 'prefix', 'metric', 'dest_router_id', 'link_prefixes', 'intra_prefixes' ]" + code => "event.set('[zeek_cols]', @zeek_ospf_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2385,23 +2387,27 @@ filter { # pe.log # https://docs.zeek.org/en/stable/scripts/base/files/pe/main.zeek.html#type-PE::Info - dissect { - id => "dissect_zeek_pe" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fuid]} %{[zeek_cols][machine]} %{[zeek_cols][compile_ts]} %{[zeek_cols][os]} %{[zeek_cols][subsystem]} %{[zeek_cols][is_exe]} %{[zeek_cols][is_64bit]} %{[zeek_cols][uses_aslr]} %{[zeek_cols][uses_dep]} %{[zeek_cols][uses_code_integrity]} %{[zeek_cols][uses_seh]} %{[zeek_cols][has_import_table]} %{[zeek_cols][has_export_table]} %{[zeek_cols][has_cert_table]} %{[zeek_cols][has_debug_data]} %{[zeek_cols][section_names]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_pe" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_pe" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fuid]} %{[zeek_cols][machine]} %{[zeek_cols][compile_ts]} %{[zeek_cols][os]} %{[zeek_cols][subsystem]} %{[zeek_cols][is_exe]} %{[zeek_cols][is_64bit]} %{[zeek_cols][uses_aslr]} %{[zeek_cols][uses_dep]} %{[zeek_cols][uses_code_integrity]} %{[zeek_cols][uses_seh]} %{[zeek_cols][has_import_table]} %{[zeek_cols][has_export_table]} %{[zeek_cols][has_cert_table]} %{[zeek_cols][has_debug_data]} %{[zeek_cols][section_names]}" + } } - ruby { - id => "ruby_zip_zeek_pe" - init => "@zeek_pe_field_names = [ 'ts', 'fuid', 'machine', 'compile_ts', 'os', 'subsystem', 'is_exe', 'is_64bit', 'uses_aslr', 'uses_dep', 'uses_code_integrity', 'uses_seh', 'has_import_table', 'has_export_table', 'has_cert_table', 'has_debug_data', 'section_names' ]" - code => "event.set('[zeek_cols]', @zeek_pe_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_pe" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_pe" + init => "@zeek_pe_field_names = [ 'ts', 'fuid', 'machine', 'compile_ts', 'os', 'subsystem', 'is_exe', 'is_64bit', 'uses_aslr', 'uses_dep', 'uses_code_integrity', 'uses_seh', 'has_import_table', 'has_export_table', 'has_cert_table', 'has_debug_data', 'section_names' ]" + code => "event.set('[zeek_cols]', @zeek_pe_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2410,23 +2416,27 @@ filter { # profinet.log # https://github.com/amzn/zeek-plugin-profinet/blob/master/scripts/main.zeek - dissect { - id => "dissect_zeek_profinet" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][operation_type]} %{[zeek_cols][block_version]} %{[zeek_cols][slot_number]} %{[zeek_cols][subslot_number]} %{[zeek_cols][index]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_profinet" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_profinet" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][operation_type]} %{[zeek_cols][block_version]} %{[zeek_cols][slot_number]} %{[zeek_cols][subslot_number]} %{[zeek_cols][index]}" + } } - ruby { - id => "ruby_zip_zeek_profinet" - init => "@zeek_profinet_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'operation_type', 'block_version', 'slot_number', 'subslot_number', 'index' ]" - code => "event.set('[zeek_cols]', @zeek_profinet_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_profinet" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_profinet" + init => "@zeek_profinet_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'operation_type', 'block_version', 'slot_number', 'subslot_number', 'index' ]" + code => "event.set('[zeek_cols]', @zeek_profinet_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2441,29 +2451,33 @@ filter { # profinet_dce_rpc.log # https://github.com/amzn/zeek-plugin-profinet/blob/master/scripts/main.zeek - dissect { - id => "dissect_zeek_profinet_dce_rpc" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][packet_type]} %{[zeek_cols][object_uuid]} %{[zeek_cols][interface_uuid]} %{[zeek_cols][activity_uuid]} %{[zeek_cols][server_boot_time]} %{[zeek_cols][operation]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_profinet_dce_rpc" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_profinet_dce_rpc" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } - } - ruby { - id => "ruby_zip_zeek_profinet_dce_rpc" - init => "@zeek_profinet_dce_rpc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'packet_type', 'object_uuid', 'interface_uuid', 'activity_uuid', 'server_boot_time', 'operation' ]" - code => "event.set('[zeek_cols]', @zeek_profinet_dce_rpc_field_names.zip(event.get('[message]')).to_h)" + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][packet_type]} %{[zeek_cols][object_uuid]} %{[zeek_cols][interface_uuid]} %{[zeek_cols][activity_uuid]} %{[zeek_cols][server_boot_time]} %{[zeek_cols][operation]}" + } } - } - - mutate { - id => "mutate_add_fields_zeek_profinet_dce_rpc" - add_field => { "[zeek_cols][service]" => "profinet_dce_rpc" } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_profinet_dce_rpc" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_profinet_dce_rpc" + init => "@zeek_profinet_dce_rpc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'packet_type', 'object_uuid', 'interface_uuid', 'activity_uuid', 'server_boot_time', 'operation' ]" + code => "event.set('[zeek_cols]', @zeek_profinet_dce_rpc_field_names.zip(event.get('[message]')).to_h)" + } + } + } + + mutate { + id => "mutate_add_fields_zeek_profinet_dce_rpc" + add_field => { "[zeek_cols][service]" => "profinet_dce_rpc" } add_tag => [ "ics" ] } @@ -2472,23 +2486,27 @@ filter { # radius.log # https://docs.zeek.org/en/stable/scripts/base/protocols/radius/main.zeek.html#type-RADIUS::Info - dissect { - id => "dissect_zeek_radius" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user]} %{[zeek_cols][mac]} %{[zeek_cols][framed_addr]} %{[zeek_cols][tunnel_client]} %{[zeek_cols][connect_info]} %{[zeek_cols][reply_msg]} %{[zeek_cols][result]} %{[zeek_cols][ttl]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_radius" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_radius" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user]} %{[zeek_cols][mac]} %{[zeek_cols][framed_addr]} %{[zeek_cols][tunnel_client]} %{[zeek_cols][connect_info]} %{[zeek_cols][reply_msg]} %{[zeek_cols][result]} %{[zeek_cols][ttl]}" + } } - ruby { - id => "ruby_zip_zeek_radius" - init => "@zeek_radius_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'mac', 'framed_addr', 'tunnel_client', 'connect_info', 'reply_msg', 'result', 'ttl' ]" - code => "event.set('[zeek_cols]', @zeek_radius_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_radius" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_radius" + init => "@zeek_radius_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'mac', 'framed_addr', 'tunnel_client', 'connect_info', 'reply_msg', 'result', 'ttl' ]" + code => "event.set('[zeek_cols]', @zeek_radius_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2502,60 +2520,27 @@ filter { # rdp.log # https://docs.zeek.org/en/stable/scripts/base/protocols/rdp/main.zeek.html#type-RDP::Info - if ([@metadata][zeek_fields_bitmap] and [@metadata][zeek_fields_bitmap_version]) { - - # bitmap rdp.log field configuration version 0 - # all fields: 0x03FFFFF / 4194303 - # minus client_channels: 0x03FFDFF / 4193791 - - if ([@metadata][zeek_fields_bitmap_version] == 0) { - - if ([@metadata][zeek_fields_bitmap] == 4194303) { - dissect { - id => "dissect_zeek_rdp_with_all_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][cookie]} %{[zeek_cols][result]} %{[zeek_cols][security_protocol]} %{[zeek_cols][client_channels]} %{[zeek_cols][keyboard_layout]} %{[zeek_cols][client_build]} %{[zeek_cols][client_name]} %{[zeek_cols][client_dig_product_id]} %{[zeek_cols][desktop_width]} %{[zeek_cols][desktop_height]} %{[zeek_cols][requested_color_depth]} %{[zeek_cols][cert_type]} %{[zeek_cols][cert_count]} %{[zeek_cols][cert_permanent]} %{[zeek_cols][encryption_level]} %{[zeek_cols][encryption_method]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 4193791) { - dissect { - id => "dissect_zeek_rdp_with_all_fields_minus_client_channels" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][cookie]} %{[zeek_cols][result]} %{[zeek_cols][security_protocol]} %{[zeek_cols][keyboard_layout]} %{[zeek_cols][client_build]} %{[zeek_cols][client_name]} %{[zeek_cols][client_dig_product_id]} %{[zeek_cols][desktop_width]} %{[zeek_cols][desktop_height]} %{[zeek_cols][requested_color_depth]} %{[zeek_cols][cert_type]} %{[zeek_cols][cert_count]} %{[zeek_cols][cert_permanent]} %{[zeek_cols][encryption_level]} %{[zeek_cols][encryption_method]}" - } - } - - } else { - # who knows? the rdp.log preprocessed bitmap is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_rdp_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - } else { - # who knows? the rdp.log preprocessed bitmap field list version is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_bitmap_rdp_version" - add_tag => [ "_dissectfailure" ] } - } + if "_jsonparsesuccess" in [tags] { } else { - # who knows? the rdp.log was not preprocessed to determine fields, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_missing_rdp_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_rdp" + dissect { + id => "dissect_zeek_rdp_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][cookie]} %{[zeek_cols][result]} %{[zeek_cols][security_protocol]} %{[zeek_cols][client_channels]} %{[zeek_cols][keyboard_layout]} %{[zeek_cols][client_build]} %{[zeek_cols][client_name]} %{[zeek_cols][client_dig_product_id]} %{[zeek_cols][desktop_width]} %{[zeek_cols][desktop_height]} %{[zeek_cols][requested_color_depth]} %{[zeek_cols][cert_type]} %{[zeek_cols][cert_count]} %{[zeek_cols][cert_permanent]} %{[zeek_cols][encryption_level]} %{[zeek_cols][encryption_method]}" + } } - ruby { - id => "ruby_zip_zeek_rdp" - init => "@zeek_rdp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'cookie', 'result', 'security_protocol', 'client_channels', 'keyboard_layout', 'client_build', 'client_name', 'client_dig_product_id', 'desktop_width', 'desktop_height', 'requested_color_depth', 'cert_type', 'cert_count', 'cert_permanent', 'encryption_level', 'encryption_method' ]" - code => "event.set('[zeek_cols]', @zeek_rdp_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_rdp" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_rdp" + init => "@zeek_rdp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'cookie', 'result', 'security_protocol', 'client_channels', 'keyboard_layout', 'client_build', 'client_name', 'client_dig_product_id', 'desktop_width', 'desktop_height', 'requested_color_depth', 'cert_type', 'cert_count', 'cert_permanent', 'encryption_level', 'encryption_method' ]" + code => "event.set('[zeek_cols]', @zeek_rdp_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2573,23 +2558,27 @@ filter { # rfb.log # https://docs.zeek.org/en/stable/scripts/base/protocols/rfb/main.zeek.html#type-RFB::Info - dissect { - id => "dissect_zeek_rfb" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][client_major_version]} %{[zeek_cols][client_minor_version]} %{[zeek_cols][server_major_version]} %{[zeek_cols][server_minor_version]} %{[zeek_cols][authentication_method]} %{[zeek_cols][auth]} %{[zeek_cols][share_flag]} %{[zeek_cols][desktop_name]} %{[zeek_cols][width]} %{[zeek_cols][height]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_rfb" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_rfb" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][client_major_version]} %{[zeek_cols][client_minor_version]} %{[zeek_cols][server_major_version]} %{[zeek_cols][server_minor_version]} %{[zeek_cols][authentication_method]} %{[zeek_cols][auth]} %{[zeek_cols][share_flag]} %{[zeek_cols][desktop_name]} %{[zeek_cols][width]} %{[zeek_cols][height]}" + } } - ruby { - id => "ruby_zip_zeek_rfb" - init => "@zeek_rfb_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'client_major_version', 'client_minor_version', 'server_major_version', 'server_minor_version', 'authentication_method', 'auth', 'share_flag', 'desktop_name', 'width', 'height' ]" - code => "event.set('[zeek_cols]', @zeek_rfb_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_rfb" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_rfb" + init => "@zeek_rfb_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'client_major_version', 'client_minor_version', 'server_major_version', 'server_minor_version', 'authentication_method', 'auth', 'share_flag', 'desktop_name', 'width', 'height' ]" + code => "event.set('[zeek_cols]', @zeek_rfb_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2603,23 +2592,27 @@ filter { # s7comm.log # https://github.com/cisagov/icsnpp-s7comm - dissect { - id => "dissect_zeek_s7comm" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][rosctr_code]} %{[zeek_cols][rosctr_name]} %{[zeek_cols][pdu_reference]} %{[zeek_cols][function_code]} %{[zeek_cols][function_name]} %{[zeek_cols][subfunction_code]} %{[zeek_cols][subfunction_name]} %{[zeek_cols][error_class]} %{[zeek_cols][error_code]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_s7comm" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_s7comm" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][rosctr_code]} %{[zeek_cols][rosctr_name]} %{[zeek_cols][pdu_reference]} %{[zeek_cols][function_code]} %{[zeek_cols][function_name]} %{[zeek_cols][subfunction_code]} %{[zeek_cols][subfunction_name]} %{[zeek_cols][error_class]} %{[zeek_cols][error_code]}" + } } - ruby { - id => "ruby_zip_zeek_s7comm" - init => "@zeek_s7comm_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'rosctr_code', 'rosctr_name', 'pdu_reference', 'function_code', 'function_name', 'subfunction_code', 'subfunction_name', 'error_class', 'error_code' ]" - code => "event.set('[zeek_cols]', @zeek_s7comm_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_s7comm" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_s7comm" + init => "@zeek_s7comm_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'rosctr_code', 'rosctr_name', 'pdu_reference', 'function_code', 'function_name', 'subfunction_code', 'subfunction_name', 'error_class', 'error_code' ]" + code => "event.set('[zeek_cols]', @zeek_s7comm_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2637,23 +2630,27 @@ filter { # s7comm_plus.log # https://github.com/cisagov/icsnpp-s7comm - dissect { - id => "dissect_zeek_s7comm_plus" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][opcode]} %{[zeek_cols][opcode_name]} %{[zeek_cols][function_code]} %{[zeek_cols][function_name]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_s7comm_plus" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_s7comm_plus" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][opcode]} %{[zeek_cols][opcode_name]} %{[zeek_cols][function_code]} %{[zeek_cols][function_name]}" + } } - ruby { - id => "ruby_zip_zeek_s7comm_plus" - init => "@zeek_s7comm_plus_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'opcode', 'opcode_name', 'function_code', 'function_name' ]" - code => "event.set('[zeek_cols]', @zeek_s7comm_plus_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_s7comm_plus" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_s7comm_plus" + init => "@zeek_s7comm_plus_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'opcode', 'opcode_name', 'function_code', 'function_name' ]" + code => "event.set('[zeek_cols]', @zeek_s7comm_plus_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2671,23 +2668,27 @@ filter { # s7comm_read_szl.log # https://github.com/cisagov/icsnpp-s7comm - dissect { - id => "dissect_zeek_s7comm_read_szl" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][pdu_reference]} %{[zeek_cols][method]} %{[zeek_cols][szl_id]} %{[zeek_cols][szl_id_name]} %{[zeek_cols][szl_index]} %{[zeek_cols][return_code]} %{[zeek_cols][return_code_name]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_s7comm_read_szl" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_s7comm_read_szl" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][pdu_reference]} %{[zeek_cols][method]} %{[zeek_cols][szl_id]} %{[zeek_cols][szl_id_name]} %{[zeek_cols][szl_index]} %{[zeek_cols][return_code]} %{[zeek_cols][return_code_name]}" + } } - ruby { - id => "ruby_zip_zeek_s7comm_read_szl" - init => "@zeek_s7comm_read_szl_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'pdu_reference', 'method', 'szl_id', 'szl_id_name', 'szl_index', 'return_code', 'return_code_name' ]" - code => "event.set('[zeek_cols]', @zeek_s7comm_read_szl_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_s7comm_read_szl" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_s7comm_read_szl" + init => "@zeek_s7comm_read_szl_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'pdu_reference', 'method', 'szl_id', 'szl_id_name', 'szl_index', 'return_code', 'return_code_name' ]" + code => "event.set('[zeek_cols]', @zeek_s7comm_read_szl_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2705,23 +2706,27 @@ filter { # s7comm_upload_download.log # https://github.com/cisagov/icsnpp-s7comm - dissect { - id => "dissect_zeek_s7comm_upload_download" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][rosctr_name]} %{[zeek_cols][pdu_reference]} %{[zeek_cols][function_name]} %{[zeek_cols][function_status]} %{[zeek_cols][session_id]} %{[zeek_cols][blocklength]} %{[zeek_cols][filename]} %{[zeek_cols][block_type]} %{[zeek_cols][block_number]} %{[zeek_cols][destination_filesystem]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_s7comm_upload_download" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_s7comm_upload_download" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][rosctr_name]} %{[zeek_cols][pdu_reference]} %{[zeek_cols][function_name]} %{[zeek_cols][function_status]} %{[zeek_cols][session_id]} %{[zeek_cols][blocklength]} %{[zeek_cols][filename]} %{[zeek_cols][block_type]} %{[zeek_cols][block_number]} %{[zeek_cols][destination_filesystem]}" + } } - ruby { - id => "ruby_zip_zeek_s7comm_upload_download" - init => "@zeek_s7comm_upload_download_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'rosctr_name', 'pdu_reference', 'function_name', 'function_status', 'session_id', 'blocklength', 'filename', 'block_type', 'block_number', 'destination_filesystem' ]" - code => "event.set('[zeek_cols]', @zeek_s7comm_upload_download_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_s7comm_upload_download" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_s7comm_upload_download" + init => "@zeek_s7comm_upload_download_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'rosctr_name', 'pdu_reference', 'function_name', 'function_status', 'session_id', 'blocklength', 'filename', 'block_type', 'block_number', 'destination_filesystem' ]" + code => "event.set('[zeek_cols]', @zeek_s7comm_upload_download_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2738,23 +2743,27 @@ filter { ############################################################################################################################# # signatures.log - dissect { - id => "dissect_zeek_signatures" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][note]} %{[zeek_cols][signature_id]} %{[zeek_cols][event_message]} %{[zeek_cols][sub_message]} %{[zeek_cols][signature_count]} %{[zeek_cols][host_count]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_signatures" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_signatures" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][note]} %{[zeek_cols][signature_id]} %{[zeek_cols][event_message]} %{[zeek_cols][sub_message]} %{[zeek_cols][signature_count]} %{[zeek_cols][host_count]}" + } } - ruby { - id => "ruby_zip_zeek_signatures" - init => "@zeek_signatures_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'note', 'signature_id', 'event_message', 'sub_message', 'signature_count', 'host_count' ]" - code => "event.set('[zeek_cols]', @zeek_signatures_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_signatures" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_signatures" + init => "@zeek_signatures_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'note', 'signature_id', 'event_message', 'sub_message', 'signature_count', 'host_count' ]" + code => "event.set('[zeek_cols]', @zeek_signatures_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2763,23 +2772,27 @@ filter { # sip.log # https://docs.zeek.org/en/stable/scripts/base/protocols/sip/main.zeek.html#type-SIP::Info - dissect { - id => "dissect_zeek_sip" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_depth]} %{[zeek_cols][method]} %{[zeek_cols][uri]} %{[zeek_cols][date]} %{[zeek_cols][request_from]} %{[zeek_cols][request_to]} %{[zeek_cols][response_from]} %{[zeek_cols][response_to]} %{[zeek_cols][reply_to]} %{[zeek_cols][call_id]} %{[zeek_cols][seq]} %{[zeek_cols][subject]} %{[zeek_cols][request_path]} %{[zeek_cols][response_path]} %{[zeek_cols][user_agent]} %{[zeek_cols][status_code]} %{[zeek_cols][status_msg]} %{[zeek_cols][warning]} %{[zeek_cols][request_body_len]} %{[zeek_cols][response_body_len]} %{[zeek_cols][content_type]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_sip" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_sip" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_depth]} %{[zeek_cols][method]} %{[zeek_cols][uri]} %{[zeek_cols][date]} %{[zeek_cols][request_from]} %{[zeek_cols][request_to]} %{[zeek_cols][response_from]} %{[zeek_cols][response_to]} %{[zeek_cols][reply_to]} %{[zeek_cols][call_id]} %{[zeek_cols][seq]} %{[zeek_cols][subject]} %{[zeek_cols][request_path]} %{[zeek_cols][response_path]} %{[zeek_cols][user_agent]} %{[zeek_cols][status_code]} %{[zeek_cols][status_msg]} %{[zeek_cols][warning]} %{[zeek_cols][request_body_len]} %{[zeek_cols][response_body_len]} %{[zeek_cols][content_type]}" + } } - ruby { - id => "ruby_zip_zeek_sip" - init => "@zeek_sip_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'method', 'uri', 'date', 'request_from', 'request_to', 'response_from', 'response_to', 'reply_to', 'call_id', 'seq', 'subject', 'request_path', 'response_path', 'user_agent', 'status_code', 'status_msg', 'warning', 'request_body_len', 'response_body_len', 'content_type' ]" - code => "event.set('[zeek_cols]', @zeek_sip_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_sip" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_sip" + init => "@zeek_sip_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'method', 'uri', 'date', 'request_from', 'request_to', 'response_from', 'response_to', 'reply_to', 'call_id', 'seq', 'subject', 'request_path', 'response_path', 'user_agent', 'status_code', 'status_msg', 'warning', 'request_body_len', 'response_body_len', 'content_type' ]" + code => "event.set('[zeek_cols]', @zeek_sip_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2796,23 +2809,27 @@ filter { # note that smb_cmd.referenced_file is exactly the same structure as the log line for smb_files. later on it will be # merged up as its own top-level entity so I don't have to duplicate the parsing effort below - dissect { - id => "dissect_zeek_smb_cmd" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][command]} %{[zeek_cols][sub_command]} %{[zeek_cols][argument]} %{[zeek_cols][status]} %{[zeek_cols][rtt]} %{[zeek_cols][version]} %{[zeek_cols][user]} %{[zeek_cols][tree]} %{[zeek_cols][tree_service]} %{[zeek_cols][referenced_file][ts]} %{[zeek_cols][referenced_file][uid]} %{[zeek_cols][referenced_file][orig_h]} %{[zeek_cols][referenced_file][orig_p]} %{[zeek_cols][referenced_file][resp_h]} %{[zeek_cols][referenced_file][resp_p]} %{[zeek_cols][referenced_file][fuid]} %{[zeek_cols][referenced_file][action]} %{[zeek_cols][referenced_file][path]} %{[zeek_cols][referenced_file][name]} %{[zeek_cols][referenced_file][size]} %{[zeek_cols][referenced_file][prev_name]} %{[zeek_cols][referenced_file][times_modified]} %{[zeek_cols][referenced_file][times_accessed]} %{[zeek_cols][referenced_file][times_created]} %{[zeek_cols][referenced_file][times_changed]} %{[zeek_cols][referenced_file][data_offset_req]} %{[zeek_cols][referenced_file][data_len_req]} %{[zeek_cols][referenced_file][data_len_rsp]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_smb_cmd" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_smb_cmd" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][command]} %{[zeek_cols][sub_command]} %{[zeek_cols][argument]} %{[zeek_cols][status]} %{[zeek_cols][rtt]} %{[zeek_cols][version]} %{[zeek_cols][user]} %{[zeek_cols][tree]} %{[zeek_cols][tree_service]} %{[zeek_cols][referenced_file][ts]} %{[zeek_cols][referenced_file][uid]} %{[zeek_cols][referenced_file][orig_h]} %{[zeek_cols][referenced_file][orig_p]} %{[zeek_cols][referenced_file][resp_h]} %{[zeek_cols][referenced_file][resp_p]} %{[zeek_cols][referenced_file][fuid]} %{[zeek_cols][referenced_file][action]} %{[zeek_cols][referenced_file][path]} %{[zeek_cols][referenced_file][name]} %{[zeek_cols][referenced_file][size]} %{[zeek_cols][referenced_file][prev_name]} %{[zeek_cols][referenced_file][times_modified]} %{[zeek_cols][referenced_file][times_accessed]} %{[zeek_cols][referenced_file][times_created]} %{[zeek_cols][referenced_file][times_changed]} %{[zeek_cols][referenced_file][data_offset_req]} %{[zeek_cols][referenced_file][data_len_req]} %{[zeek_cols][referenced_file][data_len_rsp]}" + } } - ruby { - id => "ruby_zip_zeek_smb_cmd" - init => "@zeek_smb_cmd_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'command', 'sub_command', 'argument', 'status', 'rtt', 'version', 'user', 'tree', 'tree_service', 'referenced_file.ts', 'referenced_file.uid', 'referenced_file.orig_h', 'referenced_file.orig_p', 'referenced_file.resp_h', 'referenced_file.resp_p', 'referenced_file.fuid', 'referenced_file.action', 'referenced_file.path', 'referenced_file.name', 'referenced_file.size', 'referenced_file.prev_name', 'referenced_file.times_modified', 'referenced_file.times_accessed', 'referenced_file.times_created', 'referenced_file.times_changed', 'referenced_file.data_offset_req', 'referenced_file.data_len_req', 'referenced_file.data_len_rsp' ]" - code => "event.set('[zeek_cols]', @zeek_smb_cmd_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_smb_cmd" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_smb_cmd" + init => "@zeek_smb_cmd_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'command', 'sub_command', 'argument', 'status', 'rtt', 'version', 'user', 'tree', 'tree_service', 'referenced_file.ts', 'referenced_file.uid', 'referenced_file.orig_h', 'referenced_file.orig_p', 'referenced_file.resp_h', 'referenced_file.resp_p', 'referenced_file.fuid', 'referenced_file.action', 'referenced_file.path', 'referenced_file.name', 'referenced_file.size', 'referenced_file.prev_name', 'referenced_file.times_modified', 'referenced_file.times_accessed', 'referenced_file.times_created', 'referenced_file.times_changed', 'referenced_file.data_offset_req', 'referenced_file.data_len_req', 'referenced_file.data_len_rsp' ]" + code => "event.set('[zeek_cols]', @zeek_smb_cmd_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2836,65 +2853,27 @@ filter { # smb_files.log # https://docs.zeek.org/en/stable/scripts/base/protocols/smb/main.zeek.html#type-SMB::FileInfo - if ([@metadata][zeek_fields_bitmap] and [@metadata][zeek_fields_bitmap_version]) { - - # bitmap smb_files.log field configuration version 0 - # - # smb_files.log without data_offset_req, data_len_req, data_len_rsp : 0x0000FFFF / 65535 - # smb_files.log with data_offset_req, data_len_req, data_len_rsp : 0x0007FFFF / 524287 - - # smb_files.log with data_offset_req, data_len_req, data_len_rsp - - if ([@metadata][zeek_fields_bitmap_version] == 0) { - - if ([@metadata][zeek_fields_bitmap] == 524287) { - - dissect { - id => "dissect_zeek_smb_files_with_all_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][fuid]} %{[zeek_cols][action]} %{[zeek_cols][path]} %{[zeek_cols][name]} %{[zeek_cols][size]} %{[zeek_cols][prev_name]} %{[zeek_cols][times_modified]} %{[zeek_cols][times_accessed]} %{[zeek_cols][times_created]} %{[zeek_cols][times_changed]} %{[zeek_cols][data_offset_req]} %{[zeek_cols][data_len_req]} %{[zeek_cols][data_len_rsp]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 65535) { - - dissect { - id => "dissect_zeek_smb_files_without_data_lens" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][fuid]} %{[zeek_cols][action]} %{[zeek_cols][path]} %{[zeek_cols][name]} %{[zeek_cols][size]} %{[zeek_cols][prev_name]} %{[zeek_cols][times_modified]} %{[zeek_cols][times_accessed]} %{[zeek_cols][times_created]} %{[zeek_cols][times_changed]}" - } - } - - } else { - # who knows? the smb_files.log preprocessed bitmap is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_smb_files_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - } else { - # who knows? the smb_files.log preprocessed bitmap field list version is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_bitmap_smb_files_version" - add_tag => [ "_dissectfailure" ] } - } + if "_jsonparsesuccess" in [tags] { } else { - # who knows? the smb_files.log was not preprocessed to determine fields, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_missing_smb_files_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_smb_files" + dissect { + id => "dissect_zeek_smb_files_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][fuid]} %{[zeek_cols][action]} %{[zeek_cols][path]} %{[zeek_cols][name]} %{[zeek_cols][size]} %{[zeek_cols][prev_name]} %{[zeek_cols][times_modified]} %{[zeek_cols][times_accessed]} %{[zeek_cols][times_created]} %{[zeek_cols][times_changed]} %{[zeek_cols][data_offset_req]} %{[zeek_cols][data_len_req]} %{[zeek_cols][data_len_rsp]}" + } } - ruby { - id => "ruby_zip_zeek_smb_files" - init => "@zeek_smb_files_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'fuid', 'action', 'path', 'name', 'size', 'prev_name', 'times_modified', 'times_accessed', 'times_created', 'times_changed', 'data_offset_req', 'data_len_req', 'data_len_rsp' ]" - code => "event.set('[zeek_cols]', @zeek_smb_files_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_smb_files" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_smb_files" + init => "@zeek_smb_files_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'fuid', 'action', 'path', 'name', 'size', 'prev_name', 'times_modified', 'times_accessed', 'times_created', 'times_changed', 'data_offset_req', 'data_len_req', 'data_len_rsp' ]" + code => "event.set('[zeek_cols]', @zeek_smb_files_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2914,23 +2893,27 @@ filter { # smb_mapping.log # https://docs.zeek.org/en/stable/scripts/base/protocols/smb/main.zeek.html#type-SMB::TreeInfo - dissect { - id => "dissect_zeek_smb_mapping" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][path]} %{[zeek_cols][resource_type]} %{[zeek_cols][native_file_system]} %{[zeek_cols][share_type]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_smb_mapping" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_smb_mapping" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][path]} %{[zeek_cols][resource_type]} %{[zeek_cols][native_file_system]} %{[zeek_cols][share_type]}" + } } - ruby { - id => "ruby_zip_zeek_smb_mapping" - init => "@zeek_smb_mapping_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'path', 'resource_type', 'native_file_system', 'share_type' ]" - code => "event.set('[zeek_cols]', @zeek_smb_mapping_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_smb_mapping" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_smb_mapping" + init => "@zeek_smb_mapping_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'path', 'resource_type', 'native_file_system', 'share_type' ]" + code => "event.set('[zeek_cols]', @zeek_smb_mapping_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2947,23 +2930,27 @@ filter { # smtp.log # https://docs.zeek.org/en/stable/scripts/base/protocols/smtp/main.zeek.html#type-SMTP::Info - dissect { - id => "dissect_zeek_smtp" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_depth]} %{[zeek_cols][helo]} %{[zeek_cols][mailfrom]} %{[zeek_cols][rcptto]} %{[zeek_cols][date]} %{[zeek_cols][from]} %{[zeek_cols][to]} %{[zeek_cols][cc]} %{[zeek_cols][reply_to]} %{[zeek_cols][msg_id]} %{[zeek_cols][in_reply_to]} %{[zeek_cols][subject]} %{[zeek_cols][x_originating_ip]} %{[zeek_cols][first_received]} %{[zeek_cols][second_received]} %{[zeek_cols][last_reply]} %{[zeek_cols][path]} %{[zeek_cols][user_agent]} %{[zeek_cols][tls]} %{[zeek_cols][fuid]} %{[zeek_cols][is_webmail]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_smtp" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_smtp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_depth]} %{[zeek_cols][helo]} %{[zeek_cols][mailfrom]} %{[zeek_cols][rcptto]} %{[zeek_cols][date]} %{[zeek_cols][from]} %{[zeek_cols][to]} %{[zeek_cols][cc]} %{[zeek_cols][reply_to]} %{[zeek_cols][msg_id]} %{[zeek_cols][in_reply_to]} %{[zeek_cols][subject]} %{[zeek_cols][x_originating_ip]} %{[zeek_cols][first_received]} %{[zeek_cols][second_received]} %{[zeek_cols][last_reply]} %{[zeek_cols][path]} %{[zeek_cols][user_agent]} %{[zeek_cols][tls]} %{[zeek_cols][fuid]} %{[zeek_cols][is_webmail]}" + } } - ruby { - id => "ruby_zip_zeek_smtp" - init => "@zeek_smtp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'helo', 'mailfrom', 'rcptto', 'date', 'from', 'to', 'cc', 'reply_to', 'msg_id', 'in_reply_to', 'subject', 'x_originating_ip', 'first_received', 'second_received', 'last_reply', 'path', 'user_agent', 'tls', 'fuid', 'is_webmail' ]" - code => "event.set('[zeek_cols]', @zeek_smtp_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_smtp" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_smtp" + init => "@zeek_smtp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'helo', 'mailfrom', 'rcptto', 'date', 'from', 'to', 'cc', 'reply_to', 'msg_id', 'in_reply_to', 'subject', 'x_originating_ip', 'first_received', 'second_received', 'last_reply', 'path', 'user_agent', 'tls', 'fuid', 'is_webmail' ]" + code => "event.set('[zeek_cols]', @zeek_smtp_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2980,23 +2967,27 @@ filter { # snmp.log # https://docs.zeek.org/en/stable/scripts/base/protocols/snmp/main.zeek.html#type-SNMP::Info - dissect { - id => "dissect_zeek_snmp" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][duration]} %{[zeek_cols][version]} %{[zeek_cols][community]} %{[zeek_cols][get_requests]} %{[zeek_cols][get_bulk_requests]} %{[zeek_cols][get_responses]} %{[zeek_cols][set_requests]} %{[zeek_cols][display_string]} %{[zeek_cols][up_since]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_snmp" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_snmp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][duration]} %{[zeek_cols][version]} %{[zeek_cols][community]} %{[zeek_cols][get_requests]} %{[zeek_cols][get_bulk_requests]} %{[zeek_cols][get_responses]} %{[zeek_cols][set_requests]} %{[zeek_cols][display_string]} %{[zeek_cols][up_since]}" + } } - ruby { - id => "ruby_zip_zeek_snmp" - init => "@zeek_snmp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'duration', 'version', 'community', 'get_requests', 'get_bulk_requests', 'get_responses', 'set_requests', 'display_string', 'up_since' ]" - code => "event.set('[zeek_cols]', @zeek_snmp_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_snmp" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_snmp" + init => "@zeek_snmp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'duration', 'version', 'community', 'get_requests', 'get_bulk_requests', 'get_responses', 'set_requests', 'display_string', 'up_since' ]" + code => "event.set('[zeek_cols]', @zeek_snmp_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3013,23 +3004,27 @@ filter { # socks.log # https://docs.zeek.org/en/stable/scripts/base/protocols/socks/main.zeek.html#type-SOCKS::Info - dissect { - id => "dissect_zeek_socks" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][server_status]} %{[zeek_cols][request_host]} %{[zeek_cols][request_name]} %{[zeek_cols][request_port]} %{[zeek_cols][bound_host]} %{[zeek_cols][bound_name]} %{[zeek_cols][bound_port]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_socks" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_socks" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][server_status]} %{[zeek_cols][request_host]} %{[zeek_cols][request_name]} %{[zeek_cols][request_port]} %{[zeek_cols][bound_host]} %{[zeek_cols][bound_name]} %{[zeek_cols][bound_port]}" + } } - ruby { - id => "ruby_zip_zeek_socks" - init => "@zeek_socks_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'user', 'password', 'server_status', 'request_host', 'request_name', 'request_port', 'bound_host', 'bound_name', 'bound_port' ]" - code => "event.set('[zeek_cols]', @zeek_socks_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_socks" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_socks" + init => "@zeek_socks_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'user', 'password', 'server_status', 'request_host', 'request_name', 'request_port', 'bound_host', 'bound_name', 'bound_port' ]" + code => "event.set('[zeek_cols]', @zeek_socks_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3043,23 +3038,27 @@ filter { # software.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/software/main.zeek.html#type-Software::Info - dissect { - id => "dissect_zeek_software" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][software_type]} %{[zeek_cols][name]} %{[zeek_cols][version_major]} %{[zeek_cols][version_minor]} %{[zeek_cols][version_minor2]} %{[zeek_cols][version_minor3]} %{[zeek_cols][version_addl]} %{[zeek_cols][unparsed_version]} %{[zeek_cols][url]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_software" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_software" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][software_type]} %{[zeek_cols][name]} %{[zeek_cols][version_major]} %{[zeek_cols][version_minor]} %{[zeek_cols][version_minor2]} %{[zeek_cols][version_minor3]} %{[zeek_cols][version_addl]} %{[zeek_cols][unparsed_version]} %{[zeek_cols][url]}" + } } - ruby { - id => "ruby_zip_zeek_software" - init => "@zeek_software_field_names = [ 'ts', 'orig_h', 'orig_p', 'software_type', 'name', 'version_major', 'version_minor', 'version_minor2', 'version_minor3', 'version_addl', 'unparsed_version', 'url' ]" - code => "event.set('[zeek_cols]', @zeek_software_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_software" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_software" + init => "@zeek_software_field_names = [ 'ts', 'orig_h', 'orig_p', 'software_type', 'name', 'version_major', 'version_minor', 'version_minor2', 'version_minor3', 'version_addl', 'unparsed_version', 'url' ]" + code => "event.set('[zeek_cols]', @zeek_software_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3068,23 +3067,27 @@ filter { # wireguard.log # https://github.com/corelight/zeek-spicy-wireguard/blob/master/analyzer/main.zeek - dissect { - id => "dissect_zeek_wireguard" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][established]} %{[zeek_cols][initiations]} %{[zeek_cols][responses]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_wireguard" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_wireguard" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][established]} %{[zeek_cols][initiations]} %{[zeek_cols][responses]}" + } } - ruby { - id => "ruby_zip_zeek_wireguard" - init => "@zeek_wireguard_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'established', 'initiations', 'responses' ]" - code => "event.set('[zeek_cols]', @zeek_wireguard_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_wireguard" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_wireguard" + init => "@zeek_wireguard_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'established', 'initiations', 'responses' ]" + code => "event.set('[zeek_cols]', @zeek_wireguard_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3104,63 +3107,27 @@ filter { # ssh.log # https://docs.zeek.org/en/stable/scripts/base/protocols/ssh/main.zeek.html#type-SSH::Info - if ([@metadata][zeek_fields_bitmap] and [@metadata][zeek_fields_bitmap_version]) { - - # bitmap ssh.log field configuration version 0 - # - # standard ssh.log without HASSH fingerprinting add-on script : 0x007FFFFF / 8388607 - # ssh.log with HASSH fingerprinting add-on script : 0x3FFFFFFF / 1073741823 - - if ([@metadata][zeek_fields_bitmap_version] == 0) { - - if ([@metadata][zeek_fields_bitmap] == 1073741823) { - - dissect { - id => "dissect_zeek_ssh_with_all_fields_with_hassh" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][auth_success]} %{[zeek_cols][auth_attempts]} %{[zeek_cols][direction]} %{[zeek_cols][client]} %{[zeek_cols][server]} %{[zeek_cols][cipher_alg]} %{[zeek_cols][mac_alg]} %{[zeek_cols][compression_alg]} %{[zeek_cols][kex_alg]} %{[zeek_cols][host_key_alg]} %{[zeek_cols][host_key]} %{[zeek_cols][remote_location_country_code]} %{[zeek_cols][remote_location_region]} %{[zeek_cols][remote_location_city]} %{[zeek_cols][remote_location_latitude]} %{[zeek_cols][remote_location_longitude]} %{[zeek_cols][hasshVersion]} %{[zeek_cols][hassh]} %{[zeek_cols][hasshServer]} %{[zeek_cols][cshka]} %{[zeek_cols][hasshAlgorithms]} %{[zeek_cols][sshka]} %{[zeek_cols][hasshServerAlgorithms]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 8388607) { - - dissect { - id => "dissect_zeek_ssh_with_all_fields_no_hassh" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][auth_success]} %{[zeek_cols][auth_attempts]} %{[zeek_cols][direction]} %{[zeek_cols][client]} %{[zeek_cols][server]} %{[zeek_cols][cipher_alg]} %{[zeek_cols][mac_alg]} %{[zeek_cols][compression_alg]} %{[zeek_cols][kex_alg]} %{[zeek_cols][host_key_alg]} %{[zeek_cols][host_key]} %{[zeek_cols][remote_location_country_code]} %{[zeek_cols][remote_location_region]} %{[zeek_cols][remote_location_city]} %{[zeek_cols][remote_location_latitude]} %{[zeek_cols][remote_location_longitude]}" - } - } - - } else { - # who knows? the ssh.log preprocessed bitmap is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_ssh_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - } else { - # who knows? the ssh.log preprocessed bitmap field list version is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_bitmap_ssh_version" - add_tag => [ "_dissectfailure" ] } - } + if "_jsonparsesuccess" in [tags] { } else { - # who knows? the ssh.log was not preprocessed to determine fields, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_missing_ssh_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ssh" + dissect { + id => "dissect_zeek_ssh_with_all_fields_with_hassh" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][auth_success]} %{[zeek_cols][auth_attempts]} %{[zeek_cols][direction]} %{[zeek_cols][client]} %{[zeek_cols][server]} %{[zeek_cols][cipher_alg]} %{[zeek_cols][mac_alg]} %{[zeek_cols][compression_alg]} %{[zeek_cols][kex_alg]} %{[zeek_cols][host_key_alg]} %{[zeek_cols][host_key]} %{[zeek_cols][remote_location_country_code]} %{[zeek_cols][remote_location_region]} %{[zeek_cols][remote_location_city]} %{[zeek_cols][remote_location_latitude]} %{[zeek_cols][remote_location_longitude]} %{[zeek_cols][hasshVersion]} %{[zeek_cols][hassh]} %{[zeek_cols][hasshServer]} %{[zeek_cols][cshka]} %{[zeek_cols][hasshAlgorithms]} %{[zeek_cols][sshka]} %{[zeek_cols][hasshServerAlgorithms]}" + } } - ruby { - id => "ruby_zip_zeek_ssh" - init => "@zeek_ssh_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'auth_success', 'auth_attempts', 'direction', 'client', 'server', 'cipher_alg', 'mac_alg', 'compression_alg', 'kex_alg', 'host_key_alg', 'host_key', 'remote_location_country_code', 'remote_location_region', 'remote_location_city', 'remote_location_latitude', 'remote_location_longitude', 'hasshVersion', 'hassh', 'hasshServer', 'cshka', 'hasshAlgorithms', 'sshka', 'hasshServerAlgorithms' ]" - code => "event.set('[zeek_cols]', @zeek_ssh_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ssh" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ssh" + init => "@zeek_ssh_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'auth_success', 'auth_attempts', 'direction', 'client', 'server', 'cipher_alg', 'mac_alg', 'compression_alg', 'kex_alg', 'host_key_alg', 'host_key', 'remote_location_country_code', 'remote_location_region', 'remote_location_city', 'remote_location_latitude', 'remote_location_longitude', 'hasshVersion', 'hassh', 'hasshServer', 'cshka', 'hasshAlgorithms', 'sshka', 'hasshServerAlgorithms' ]" + code => "event.set('[zeek_cols]', @zeek_ssh_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3177,63 +3144,27 @@ filter { # ssl.log # https://docs.zeek.org/en/stable/scripts/base/protocols/ssl/main.zeek.html#type-SSL::Info - if ([@metadata][zeek_fields_bitmap] and [@metadata][zeek_fields_bitmap_version]) { - - # bitmap ssl.log (v4.1.0+) field configuration version 0 - # - # standard ssl.log fields without add-on JA3 fingerprinting script : 0x0007FFFF / 524287 - # ssl.log fields with add-on JA3 fingerprinting script : 0x001FFFFF / 2097151 - - if ([@metadata][zeek_fields_bitmap_version] == 0) { - - if ([@metadata][zeek_fields_bitmap] == 2097151) { - - dissect { - id => "dissect_zeek_ssl_v1_with_ja3" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][ssl_version]} %{[zeek_cols][cipher]} %{[zeek_cols][curve]} %{[zeek_cols][server_name]} %{[zeek_cols][resumed]} %{[zeek_cols][last_alert]} %{[zeek_cols][next_protocol]} %{[zeek_cols][established]} %{[zeek_cols][ssl_history]} %{[zeek_cols][cert_chain_fps]} %{[zeek_cols][client_cert_chain_fps]} %{[zeek_cols][sni_matches_cert]} %{[zeek_cols][validation_status]} %{[zeek_cols][ja3]} %{[zeek_cols][ja3s]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 524287) { - - dissect { - id => "dissect_zeek_ssl_v1_without_ja3" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][ssl_version]} %{[zeek_cols][cipher]} %{[zeek_cols][curve]} %{[zeek_cols][server_name]} %{[zeek_cols][resumed]} %{[zeek_cols][last_alert]} %{[zeek_cols][next_protocol]} %{[zeek_cols][established]} %{[zeek_cols][ssl_history]} %{[zeek_cols][cert_chain_fps]} %{[zeek_cols][client_cert_chain_fps]} %{[zeek_cols][sni_matches_cert]} %{[zeek_cols][validation_status]}" - } - } - - } else { - # who knows? the ssl.log preprocessed bitmap is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_ssl_v1_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - } else { - # who knows? the ssl.log preprocessed bitmap field list version is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_bitmap_ssl_version" - add_tag => [ "_dissectfailure" ] } - } + if "_jsonparsesuccess" in [tags] { } else { - # who knows? the ssl.log was not preprocessed to determine fields, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_missing_ssl_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ssl" + dissect { + id => "dissect_zeek_ssl_v1_with_ja3" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][ssl_version]} %{[zeek_cols][cipher]} %{[zeek_cols][curve]} %{[zeek_cols][server_name]} %{[zeek_cols][resumed]} %{[zeek_cols][last_alert]} %{[zeek_cols][next_protocol]} %{[zeek_cols][established]} %{[zeek_cols][ssl_history]} %{[zeek_cols][cert_chain_fps]} %{[zeek_cols][client_cert_chain_fps]} %{[zeek_cols][sni_matches_cert]} %{[zeek_cols][validation_status]} %{[zeek_cols][ja3]} %{[zeek_cols][ja3s]}" + } } - ruby { - id => "ruby_zip_zeek_ssl" - init => "@zeek_ssl_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'ssl_version', 'cipher', 'curve', 'server_name', 'resumed', 'last_alert', 'next_protocol', 'established', 'ssl_history', 'cert_chain_fps', 'client_cert_chain_fps', 'sni_matches_cert', 'validation_status', 'ja3', 'ja3s' ]" - code => "event.set('[zeek_cols]', @zeek_ssl_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ssl" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ssl" + init => "@zeek_ssl_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'ssl_version', 'cipher', 'curve', 'server_name', 'resumed', 'last_alert', 'next_protocol', 'established', 'ssl_history', 'cert_chain_fps', 'client_cert_chain_fps', 'sni_matches_cert', 'validation_status', 'ja3', 'ja3s' ]" + code => "event.set('[zeek_cols]', @zeek_ssl_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3250,23 +3181,27 @@ filter { # stun.log # https://github.com/corelight/zeek-spicy-stun/blob/master/analyzer/main.zeek - dissect { - id => "dissect_zeek_stun" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][is_orig]} %{[zeek_cols][trans_id]} %{[zeek_cols][method]} %{[zeek_cols][class]} %{[zeek_cols][attr_type]} %{[zeek_cols][attr_val]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_stun" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_stun" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][is_orig]} %{[zeek_cols][trans_id]} %{[zeek_cols][method]} %{[zeek_cols][class]} %{[zeek_cols][attr_type]} %{[zeek_cols][attr_val]}" + } } - ruby { - id => "ruby_zip_zeek_stun" - init => "@zeek_stun_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'is_orig', 'trans_id', 'method', 'class', 'attr_type', 'attr_val' ]" - code => "event.set('[zeek_cols]', @zeek_stun_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_stun" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_stun" + init => "@zeek_stun_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'is_orig', 'trans_id', 'method', 'class', 'attr_type', 'attr_val' ]" + code => "event.set('[zeek_cols]', @zeek_stun_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3282,23 +3217,27 @@ filter { # stun.log # https://github.com/corelight/zeek-spicy-stun/blob/master/analyzer/main.zeek - dissect { - id => "dissect_zeek_stun_nat" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][is_orig]} %{[zeek_cols][wan_addr]} %{[zeek_cols][wan_port]} %{[zeek_cols][lan_addr]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_stun_nat" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_stun_nat" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][is_orig]} %{[zeek_cols][wan_addr]} %{[zeek_cols][wan_port]} %{[zeek_cols][lan_addr]}" + } } - ruby { - id => "ruby_zip_zeek_stun_nat" - init => "@zeek_stun_nat_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'is_orig', 'wan_addr', 'wan_port', 'lan_addr' ]" - code => "event.set('[zeek_cols]', @zeek_stun_nat_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_stun_nat" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_stun_nat" + init => "@zeek_stun_nat_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'is_orig', 'wan_addr', 'wan_port', 'lan_addr' ]" + code => "event.set('[zeek_cols]', @zeek_stun_nat_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3314,24 +3253,27 @@ filter { # synchrophasor.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - dissect { - id => "dissect_zeek_synchrophasor" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][version]} %{[zeek_cols][data_stream_id]} %{[zeek_cols][history]} %{[zeek_cols][frame_size_min]} %{[zeek_cols][frame_size_max]} %{[zeek_cols][frame_size_tot]} %{[zeek_cols][data_frame_count]} %{[zeek_cols][data_rate]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_synchrophasor" + } else { + dissect { + id => "dissect_zeek_synchrophasor" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][version]} %{[zeek_cols][data_stream_id]} %{[zeek_cols][history]} %{[zeek_cols][frame_size_min]} %{[zeek_cols][frame_size_max]} %{[zeek_cols][frame_size_tot]} %{[zeek_cols][data_frame_count]} %{[zeek_cols][data_rate]}" + } } - ruby { - id => "ruby_zip_zeek_synchrophasor" - init => "@zeek_synchrophasor_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'version', 'data_stream_id', 'history', 'frame_size_min', 'frame_size_max', 'frame_size_tot', 'data_frame_count', 'data_rate' ]" - code => "event.set('[zeek_cols]', @zeek_synchrophasor_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_synchrophasor" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_synchrophasor" + init => "@zeek_synchrophasor_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'version', 'data_stream_id', 'history', 'frame_size_min', 'frame_size_max', 'frame_size_tot', 'data_frame_count', 'data_rate' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3349,24 +3291,27 @@ filter { # synchrophasor_cmd.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - dissect { - id => "dissect_zeek_synchrophasor_cmd" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][frame_size]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][command]} %{[zeek_cols][extframe]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_synchrophasor_cmd" + } else { + dissect { + id => "dissect_zeek_synchrophasor_cmd" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][frame_size]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][command]} %{[zeek_cols][extframe]}" + } } - ruby { - id => "ruby_zip_zeek_synchrophasor_cmd" - init => "@zeek_synchrophasor_cmd_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'command', 'extframe' ]" - code => "event.set('[zeek_cols]', @zeek_synchrophasor_cmd_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_synchrophasor_cmd" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_synchrophasor_cmd" + init => "@zeek_synchrophasor_cmd_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'command', 'extframe' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_cmd_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3383,24 +3328,27 @@ filter { # synchrophasor_cfg.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - dissect { - id => "dissect_zeek_synchrophasor_cfg" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][frame_size]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][cont_idx]} %{[zeek_cols][pmu_count_expected]} %{[zeek_cols][pmu_count_actual]} %{[zeek_cols][data_rate]} %{[zeek_cols][cfg_frame_id]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_synchrophasor_cfg" + } else { + dissect { + id => "dissect_zeek_synchrophasor_cfg" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][frame_size]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][cont_idx]} %{[zeek_cols][pmu_count_expected]} %{[zeek_cols][pmu_count_actual]} %{[zeek_cols][data_rate]} %{[zeek_cols][cfg_frame_id]}" + } } - ruby { - id => "ruby_zip_zeek_synchrophasor_cfg" - init => "@zeek_synchrophasor_cfg_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'cont_idx', 'pmu_count_expected', 'pmu_count_actual', 'data_rate', 'cfg_frame_id' ]" - code => "event.set('[zeek_cols]', @zeek_synchrophasor_cfg_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_synchrophasor_cfg" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_synchrophasor_cfg" + init => "@zeek_synchrophasor_cfg_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'cont_idx', 'pmu_count_expected', 'pmu_count_actual', 'data_rate', 'cfg_frame_id' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_cfg_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3417,24 +3365,27 @@ filter { # synchrophasor_cfg_detail.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - dissect { - id => "dissect_zeek_synchrophasor_cfg_detail" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][cfg_frame_id]} %{[zeek_cols][pmu_idx]} %{[zeek_cols][svc_class]} %{[zeek_cols][station_name]} %{[zeek_cols][data_source_id]} %{[zeek_cols][global_pmuid]} %{[zeek_cols][phasor_shape]} %{[zeek_cols][phasor_format]} %{[zeek_cols][analog_format]} %{[zeek_cols][freq_format]} %{[zeek_cols][phnmr]} %{[zeek_cols][annmr]} %{[zeek_cols][dgnmr]} %{[zeek_cols][phnam]} %{[zeek_cols][annam]} %{[zeek_cols][dgnam]} %{[zeek_cols][phasor_conv_phunit]} %{[zeek_cols][phasor_conv_phvalue]} %{[zeek_cols][phasor_conv_upsampled_interpolation]} %{[zeek_cols][phasor_conv_upsampled_extrapolation]} %{[zeek_cols][phasor_conv_downsampled_reselection]} %{[zeek_cols][phasor_conv_downsampled_fir_filter]} %{[zeek_cols][phasor_conv_downsampled_no_fir_filter]} %{[zeek_cols][phasor_conv_filtered_without_changing_sampling]} %{[zeek_cols][phasor_conv_calibration_mag_adj]} %{[zeek_cols][phasor_conv_calibration_phas_adj]} %{[zeek_cols][phasor_conv_rotation_phase_adj]} %{[zeek_cols][phasor_conv_pseudo_phasor_val]} %{[zeek_cols][phasor_conv_mod_appl]} %{[zeek_cols][phasor_conv_phasor_component]} %{[zeek_cols][phasor_conv_phasor_type]} %{[zeek_cols][phasor_conv_user_def]} %{[zeek_cols][phasor_conv_scale_factor]} %{[zeek_cols][phasor_conv_angle_adj]} %{[zeek_cols][analog_conv_analog_flags]} %{[zeek_cols][analog_conv_user_defined_scaling]} %{[zeek_cols][analog_conv_mag_scale]} %{[zeek_cols][analog_conv_offset]} %{[zeek_cols][digital_conv_normal_status_mask]} %{[zeek_cols][digital_conv_valid_inputs_mask]} %{[zeek_cols][pmu_lat]} %{[zeek_cols][pmu_lon]} %{[zeek_cols][pmu_elev]} %{[zeek_cols][window]} %{[zeek_cols][group_delay]} %{[zeek_cols][fnom]} %{[zeek_cols][cfgcnt]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_synchrophasor_cfg_detail" + } else { + dissect { + id => "dissect_zeek_synchrophasor_cfg_detail" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][cfg_frame_id]} %{[zeek_cols][pmu_idx]} %{[zeek_cols][svc_class]} %{[zeek_cols][station_name]} %{[zeek_cols][data_source_id]} %{[zeek_cols][global_pmuid]} %{[zeek_cols][phasor_shape]} %{[zeek_cols][phasor_format]} %{[zeek_cols][analog_format]} %{[zeek_cols][freq_format]} %{[zeek_cols][phnmr]} %{[zeek_cols][annmr]} %{[zeek_cols][dgnmr]} %{[zeek_cols][phnam]} %{[zeek_cols][annam]} %{[zeek_cols][dgnam]} %{[zeek_cols][phasor_conv_phunit]} %{[zeek_cols][phasor_conv_phvalue]} %{[zeek_cols][phasor_conv_upsampled_interpolation]} %{[zeek_cols][phasor_conv_upsampled_extrapolation]} %{[zeek_cols][phasor_conv_downsampled_reselection]} %{[zeek_cols][phasor_conv_downsampled_fir_filter]} %{[zeek_cols][phasor_conv_downsampled_no_fir_filter]} %{[zeek_cols][phasor_conv_filtered_without_changing_sampling]} %{[zeek_cols][phasor_conv_calibration_mag_adj]} %{[zeek_cols][phasor_conv_calibration_phas_adj]} %{[zeek_cols][phasor_conv_rotation_phase_adj]} %{[zeek_cols][phasor_conv_pseudo_phasor_val]} %{[zeek_cols][phasor_conv_mod_appl]} %{[zeek_cols][phasor_conv_phasor_component]} %{[zeek_cols][phasor_conv_phasor_type]} %{[zeek_cols][phasor_conv_user_def]} %{[zeek_cols][phasor_conv_scale_factor]} %{[zeek_cols][phasor_conv_angle_adj]} %{[zeek_cols][analog_conv_analog_flags]} %{[zeek_cols][analog_conv_user_defined_scaling]} %{[zeek_cols][analog_conv_mag_scale]} %{[zeek_cols][analog_conv_offset]} %{[zeek_cols][digital_conv_normal_status_mask]} %{[zeek_cols][digital_conv_valid_inputs_mask]} %{[zeek_cols][pmu_lat]} %{[zeek_cols][pmu_lon]} %{[zeek_cols][pmu_elev]} %{[zeek_cols][window]} %{[zeek_cols][group_delay]} %{[zeek_cols][fnom]} %{[zeek_cols][cfgcnt]}" + } } - ruby { - id => "ruby_zip_zeek_synchrophasor_cfg_detail" - init => "@zeek_synchrophasor_cfg_detail_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'header_time_stamp', 'cfg_frame_id', 'pmu_idx', 'svc_class', 'station_name', 'data_source_id', 'global_pmuid', 'phasor_shape', 'phasor_format', 'analog_format', 'freq_format', 'phnmr', 'annmr', 'dgnmr', 'phnam', 'annam', 'dgnam', 'phasor_conv_phunit', 'phasor_conv_phvalue', 'phasor_conv_upsampled_interpolation', 'phasor_conv_upsampled_extrapolation', 'phasor_conv_downsampled_reselection', 'phasor_conv_downsampled_fir_filter', 'phasor_conv_downsampled_no_fir_filter', 'phasor_conv_filtered_without_changing_sampling', 'phasor_conv_calibration_mag_adj', 'phasor_conv_calibration_phas_adj', 'phasor_conv_rotation_phase_adj', 'phasor_conv_pseudo_phasor_val', 'phasor_conv_mod_appl', 'phasor_conv_phasor_component', 'phasor_conv_phasor_type', 'phasor_conv_user_def', 'phasor_conv_scale_factor', 'phasor_conv_angle_adj', 'analog_conv_analog_flags', 'analog_conv_user_defined_scaling', 'analog_conv_mag_scale', 'analog_conv_offset', 'digital_conv_normal_status_mask', 'digital_conv_valid_inputs_mask', 'pmu_lat', 'pmu_lon', 'pmu_elev', 'window', 'group_delay', 'fnom', 'cfgcnt' ]" - code => "event.set('[zeek_cols]', @zeek_synchrophasor_cfg_detail_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_synchrophasor_cfg_detail" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_synchrophasor_cfg_detail" + init => "@zeek_synchrophasor_cfg_detail_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'header_time_stamp', 'cfg_frame_id', 'pmu_idx', 'svc_class', 'station_name', 'data_source_id', 'global_pmuid', 'phasor_shape', 'phasor_format', 'analog_format', 'freq_format', 'phnmr', 'annmr', 'dgnmr', 'phnam', 'annam', 'dgnam', 'phasor_conv_phunit', 'phasor_conv_phvalue', 'phasor_conv_upsampled_interpolation', 'phasor_conv_upsampled_extrapolation', 'phasor_conv_downsampled_reselection', 'phasor_conv_downsampled_fir_filter', 'phasor_conv_downsampled_no_fir_filter', 'phasor_conv_filtered_without_changing_sampling', 'phasor_conv_calibration_mag_adj', 'phasor_conv_calibration_phas_adj', 'phasor_conv_rotation_phase_adj', 'phasor_conv_pseudo_phasor_val', 'phasor_conv_mod_appl', 'phasor_conv_phasor_component', 'phasor_conv_phasor_type', 'phasor_conv_user_def', 'phasor_conv_scale_factor', 'phasor_conv_angle_adj', 'analog_conv_analog_flags', 'analog_conv_user_defined_scaling', 'analog_conv_mag_scale', 'analog_conv_offset', 'digital_conv_normal_status_mask', 'digital_conv_valid_inputs_mask', 'pmu_lat', 'pmu_lon', 'pmu_elev', 'window', 'group_delay', 'fnom', 'cfgcnt' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_cfg_detail_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3451,24 +3402,27 @@ filter { # synchrophasor_data.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - dissect { - id => "dissect_zeek_synchrophasor_data" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][frame_size]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][pmu_count_expected]} %{[zeek_cols][pmu_count_actual]} %{[zeek_cols][data_frame_id]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_synchrophasor_data" + } else { + dissect { + id => "dissect_zeek_synchrophasor_data" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][frame_size]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][pmu_count_expected]} %{[zeek_cols][pmu_count_actual]} %{[zeek_cols][data_frame_id]}" + } } - ruby { - id => "ruby_zip_zeek_synchrophasor_data" - init => "@zeek_synchrophasor_data_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'pmu_count_expected', 'pmu_count_actual', 'data_frame_id' ]" - code => "event.set('[zeek_cols]', @zeek_synchrophasor_data_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_synchrophasor_data" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_synchrophasor_data" + init => "@zeek_synchrophasor_data_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'pmu_count_expected', 'pmu_count_actual', 'data_frame_id' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_data_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3485,24 +3439,27 @@ filter { # synchrophasor_data_detail.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - dissect { - id => "dissect_zeek_synchrophasor_data_detail" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][data_frame_id]} %{[zeek_cols][pmu_idx]} %{[zeek_cols][trigger_reason]} %{[zeek_cols][unlocked_time]} %{[zeek_cols][pmu_time_quality]} %{[zeek_cols][data_modified]} %{[zeek_cols][config_change]} %{[zeek_cols][pmu_trigger_pickup]} %{[zeek_cols][data_sorting_type]} %{[zeek_cols][pmu_sync_error]} %{[zeek_cols][data_error_indicator]} %{[zeek_cols][est_rectangular_real]} %{[zeek_cols][est_rectangular_imaginary]} %{[zeek_cols][est_polar_magnitude]} %{[zeek_cols][est_polar_angle]} %{[zeek_cols][freq_dev_mhz]} %{[zeek_cols][rocof]} %{[zeek_cols][analog_data]} %{[zeek_cols][digital]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_synchrophasor_data_detail" + } else { + dissect { + id => "dissect_zeek_synchrophasor_data_detail" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][data_frame_id]} %{[zeek_cols][pmu_idx]} %{[zeek_cols][trigger_reason]} %{[zeek_cols][unlocked_time]} %{[zeek_cols][pmu_time_quality]} %{[zeek_cols][data_modified]} %{[zeek_cols][config_change]} %{[zeek_cols][pmu_trigger_pickup]} %{[zeek_cols][data_sorting_type]} %{[zeek_cols][pmu_sync_error]} %{[zeek_cols][data_error_indicator]} %{[zeek_cols][est_rectangular_real]} %{[zeek_cols][est_rectangular_imaginary]} %{[zeek_cols][est_polar_magnitude]} %{[zeek_cols][est_polar_angle]} %{[zeek_cols][freq_dev_mhz]} %{[zeek_cols][rocof]} %{[zeek_cols][analog_data]} %{[zeek_cols][digital]}" + } } - ruby { - id => "ruby_zip_zeek_synchrophasor_data_detail" - init => "@zeek_synchrophasor_data_detail_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'header_time_stamp', 'data_frame_id', 'pmu_idx', 'trigger_reason', 'unlocked_time', 'pmu_time_quality', 'data_modified', 'config_change', 'pmu_trigger_pickup', 'data_sorting_type', 'pmu_sync_error', 'data_error_indicator', 'est_rectangular_real', 'est_rectangular_imaginary', 'est_polar_magnitude', 'est_polar_angle', 'freq_dev_mhz', 'rocof', 'analog_data', 'digital' ]" - code => "event.set('[zeek_cols]', @zeek_synchrophasor_data_detail_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_synchrophasor_data_detail" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_synchrophasor_data_detail" + init => "@zeek_synchrophasor_data_detail_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'header_time_stamp', 'data_frame_id', 'pmu_idx', 'trigger_reason', 'unlocked_time', 'pmu_time_quality', 'data_modified', 'config_change', 'pmu_trigger_pickup', 'data_sorting_type', 'pmu_sync_error', 'data_error_indicator', 'est_rectangular_real', 'est_rectangular_imaginary', 'est_polar_magnitude', 'est_polar_angle', 'freq_dev_mhz', 'rocof', 'analog_data', 'digital' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_data_detail_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3519,24 +3476,27 @@ filter { # synchrophasor_hdr.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - dissect { - id => "dissect_zeek_synchrophasor_hdr" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][frame_size]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][data]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_synchrophasor_hdr" + } else { + dissect { + id => "dissect_zeek_synchrophasor_hdr" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][frame_size]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][data]}" + } } - ruby { - id => "ruby_zip_zeek_synchrophasor_hdr" - init => "@zeek_synchrophasor_hdr_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'data' ]" - code => "event.set('[zeek_cols]', @zeek_synchrophasor_hdr_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_synchrophasor_hdr" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_synchrophasor_hdr" + init => "@zeek_synchrophasor_hdr_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'data' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_hdr_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3553,23 +3513,27 @@ filter { # syslog.log # https://docs.zeek.org/en/stable/scripts/base/protocols/syslog/main.zeek.html#type-Syslog::Info - dissect { - id => "dissect_zeek_syslog" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][facility]} %{[zeek_cols][severity]} %{[zeek_cols][message]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_syslog" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_syslog" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][facility]} %{[zeek_cols][severity]} %{[zeek_cols][message]}" + } } - ruby { - id => "ruby_zip_zeek_syslog" - init => "@zeek_syslog_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'facility', 'severity', 'message' ]" - code => "event.set('[zeek_cols]', @zeek_syslog_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_syslog" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_syslog" + init => "@zeek_syslog_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'facility', 'severity', 'message' ]" + code => "event.set('[zeek_cols]', @zeek_syslog_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3583,23 +3547,27 @@ filter { # tds.log # https://github.com/amzn/zeek-plugin-tds/blob/master/scripts/main.zeek - dissect { - id => "dissect_zeek_tds" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][command]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_tds" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_tds" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][command]}" + } } - ruby { - id => "ruby_zip_zeek_tds" - init => "@zeek_tds_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'command' ]" - code => "event.set('[zeek_cols]', @zeek_tds_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_tds" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_tds" + init => "@zeek_tds_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'command' ]" + code => "event.set('[zeek_cols]', @zeek_tds_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3616,23 +3584,27 @@ filter { # tds_rpc.log # https://github.com/amzn/zeek-plugin-tds/blob/master/scripts/main.zeek - dissect { - id => "dissect_zeek_tds_rpc" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][procedure_name]} %{[zeek_cols][parameter]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_tds_rpc" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_tds_rpc" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][procedure_name]} %{[zeek_cols][parameter]}" + } } - ruby { - id => "ruby_zip_zeek_tds_rpc" - init => "@zeek_tds_rpc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'procedure_name', 'parameter' ]" - code => "event.set('[zeek_cols]', @zeek_tds_rpc_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_tds_rpc" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_tds_rpc" + init => "@zeek_tds_rpc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'procedure_name', 'parameter' ]" + code => "event.set('[zeek_cols]', @zeek_tds_rpc_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3649,23 +3621,27 @@ filter { # tds_sql_batch.log # https://github.com/amzn/zeek-plugin-tds/blob/master/scripts/main.zeek - dissect { - id => "dissect_zeek_tds_sql_batch" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][header_type]} %{[zeek_cols][query]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_tds_sql_batch" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_tds_sql_batch" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][header_type]} %{[zeek_cols][query]}" + } } - ruby { - id => "ruby_zip_zeek_tds_sql_batch" - init => "@zeek_tds_sql_batch_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'header_type', 'query' ]" - code => "event.set('[zeek_cols]', @zeek_tds_sql_batch_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_tds_sql_batch" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_tds_sql_batch" + init => "@zeek_tds_sql_batch_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'header_type', 'query' ]" + code => "event.set('[zeek_cols]', @zeek_tds_sql_batch_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3682,23 +3658,27 @@ filter { # tftp.log # https://github.com/zeek/spicy-tftp - dissect { - id => "dissect_zeek_tftp" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][wrq]} %{[zeek_cols][fname]} %{[zeek_cols][mode]} %{[zeek_cols][uid_data]} %{[zeek_cols][size]} %{[zeek_cols][block_sent]} %{[zeek_cols][block_acked]} %{[zeek_cols][error_code]} %{[zeek_cols][error_msg]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_tftp" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_tftp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][wrq]} %{[zeek_cols][fname]} %{[zeek_cols][mode]} %{[zeek_cols][uid_data]} %{[zeek_cols][size]} %{[zeek_cols][block_sent]} %{[zeek_cols][block_acked]} %{[zeek_cols][error_code]} %{[zeek_cols][error_msg]}" + } } - ruby { - id => "ruby_zip_zeek_tftp" - init => "@zeek_tftp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'wrq', 'fname', 'mode', 'uid_data', 'size', 'block_sent', 'block_acked', 'error_code', 'error_msg' ]" - code => "event.set('[zeek_cols]', @zeek_tftp_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_tftp" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_tftp" + init => "@zeek_tftp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'wrq', 'fname', 'mode', 'uid_data', 'size', 'block_sent', 'block_acked', 'error_code', 'error_msg' ]" + code => "event.set('[zeek_cols]', @zeek_tftp_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3715,23 +3695,27 @@ filter { # tunnel.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/tunnels/main.zeek.html#type-Tunnel::Info - dissect { - id => "dissect_zeek_tunnel" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][tunnel_type]} %{[zeek_cols][action]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_tunnel" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_tunnel" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][tunnel_type]} %{[zeek_cols][action]}" + } } - ruby { - id => "ruby_zip_zeek_tunnel" - init => "@zeek_tunnel_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'tunnel_type', 'action' ]" - code => "event.set('[zeek_cols]', @zeek_tunnel_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_tunnel" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_tunnel" + init => "@zeek_tunnel_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'tunnel_type', 'action' ]" + code => "event.set('[zeek_cols]', @zeek_tunnel_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3754,23 +3738,27 @@ filter { # weird.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/notice/weird.zeek.html#type-Weird::Info - dissect { - id => "dissect_zeek_weird" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][name]} %{[zeek_cols][addl]} %{[zeek_cols][notice]} %{[zeek_cols][peer]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_weird" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_weird" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][name]} %{[zeek_cols][addl]} %{[zeek_cols][notice]} %{[zeek_cols][peer]}" + } } - ruby { - id => "ruby_zip_zeek_weird" - init => "@zeek_weird_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'name', 'addl', 'notice', 'peer' ]" - code => "event.set('[zeek_cols]', @zeek_weird_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_weird" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_weird" + init => "@zeek_weird_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'name', 'addl', 'notice', 'peer' ]" + code => "event.set('[zeek_cols]', @zeek_weird_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3779,24 +3767,27 @@ filter { # x509.log # https://docs.zeek.org/en/stable/scripts/base/files/x509/main.zeek.html#type-X509::Info - dissect { - id => "dissect_zeek_x509_v1" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fingerprint]} %{[zeek_cols][certificate_version]} %{[zeek_cols][certificate_serial]} %{[zeek_cols][certificate_subject]} %{[zeek_cols][certificate_issuer]} %{[zeek_cols][certificate_not_valid_before]} %{[zeek_cols][certificate_not_valid_after]} %{[zeek_cols][certificate_key_alg]} %{[zeek_cols][certificate_sig_alg]} %{[zeek_cols][certificate_key_type]} %{[zeek_cols][certificate_key_length]} %{[zeek_cols][certificate_exponent]} %{[zeek_cols][certificate_curve]} %{[zeek_cols][san_dns]} %{[zeek_cols][san_uri]} %{[zeek_cols][san_email]} %{[zeek_cols][san_ip]} %{[zeek_cols][basic_constraints_ca]} %{[zeek_cols][basic_constraints_path_len]} %{[zeek_cols][host_cert]} %{[zeek_cols][client_cert]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_x509" + } else { + dissect { + id => "dissect_zeek_x509_v1" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fingerprint]} %{[zeek_cols][certificate_version]} %{[zeek_cols][certificate_serial]} %{[zeek_cols][certificate_subject]} %{[zeek_cols][certificate_issuer]} %{[zeek_cols][certificate_not_valid_before]} %{[zeek_cols][certificate_not_valid_after]} %{[zeek_cols][certificate_key_alg]} %{[zeek_cols][certificate_sig_alg]} %{[zeek_cols][certificate_key_type]} %{[zeek_cols][certificate_key_length]} %{[zeek_cols][certificate_exponent]} %{[zeek_cols][certificate_curve]} %{[zeek_cols][san_dns]} %{[zeek_cols][san_uri]} %{[zeek_cols][san_email]} %{[zeek_cols][san_ip]} %{[zeek_cols][basic_constraints_ca]} %{[zeek_cols][basic_constraints_path_len]} %{[zeek_cols][host_cert]} %{[zeek_cols][client_cert]}" + } } - ruby { - id => "ruby_zip_zeek_x509" - init => "@zeek_x509_field_names = [ 'ts', 'fuid', 'certificate_version', 'certificate_serial', 'certificate_subject', 'certificate_issuer', 'certificate_not_valid_before', 'certificate_not_valid_after', 'certificate_key_alg', 'certificate_sig_alg', 'certificate_key_type', 'certificate_key_length', 'certificate_exponent', 'certificate_curve', 'san_dns', 'san_uri', 'san_email', 'san_ip', 'basic_constraints_ca', 'basic_constraints_path_len', 'host_cert', 'client_cert' ]" - code => "event.set('[zeek_cols]', @zeek_x509_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_x509" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_x509" + init => "@zeek_x509_field_names = [ 'ts', 'fuid', 'certificate_version', 'certificate_serial', 'certificate_subject', 'certificate_issuer', 'certificate_not_valid_before', 'certificate_not_valid_after', 'certificate_key_alg', 'certificate_sig_alg', 'certificate_key_type', 'certificate_key_length', 'certificate_exponent', 'certificate_curve', 'san_dns', 'san_uri', 'san_email', 'san_ip', 'basic_constraints_ca', 'basic_constraints_path_len', 'host_cert', 'client_cert' ]" + code => "event.set('[zeek_cols]', @zeek_x509_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3810,24 +3801,27 @@ filter { # opcua_binary.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][msg_type]} %{[zeek_cols][is_final]} %{[zeek_cols][msg_size]} %{[zeek_cols][error]} %{[zeek_cols][reason]} %{[zeek_cols][version]} %{[zeek_cols][rcv_buf_size]} %{[zeek_cols][snd_buf_size]} %{[zeek_cols][max_msg_size]} %{[zeek_cols][max_chunk_cnt]} %{[zeek_cols][endpoint_url]} %{[zeek_cols][sec_channel_id]} %{[zeek_cols][sec_policy_uri_len]} %{[zeek_cols][sec_policy_uri]} %{[zeek_cols][snd_cert_len]} %{[zeek_cols][snd_cert]} %{[zeek_cols][rcv_cert_len]} %{[zeek_cols][rcv_cert]} %{[zeek_cols][seq_number]} %{[zeek_cols][request_id]} %{[zeek_cols][encoding_mask]} %{[zeek_cols][namespace_idx]} %{[zeek_cols][identifier]} %{[zeek_cols][identifier_str]} %{[zeek_cols][req_hdr_node_id_type]} %{[zeek_cols][req_hdr_node_id_namespace_idx]} %{[zeek_cols][req_hdr_node_id_numeric]} %{[zeek_cols][req_hdr_node_id_string]} %{[zeek_cols][req_hdr_node_id_guid]} %{[zeek_cols][req_hdr_node_id_opaque]} %{[zeek_cols][req_hdr_timestamp]} %{[zeek_cols][req_hdr_request_handle]} %{[zeek_cols][req_hdr_return_diag]} %{[zeek_cols][req_hdr_audit_entry_id]} %{[zeek_cols][req_hdr_timeout_hint]} %{[zeek_cols][req_hdr_add_hdr_type_id]} %{[zeek_cols][req_hdr_add_hdr_enc_mask]} %{[zeek_cols][res_hdr_timestamp]} %{[zeek_cols][res_hdr_request_handle]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][res_hdr_service_diag_encoding]} %{[zeek_cols][res_hdr_add_hdr_type_id]} %{[zeek_cols][res_hdr_add_hdr_enc_mask]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary" + } else { + dissect { + id => "dissect_zeek_opcua_binary" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][msg_type]} %{[zeek_cols][is_final]} %{[zeek_cols][msg_size]} %{[zeek_cols][error]} %{[zeek_cols][reason]} %{[zeek_cols][version]} %{[zeek_cols][rcv_buf_size]} %{[zeek_cols][snd_buf_size]} %{[zeek_cols][max_msg_size]} %{[zeek_cols][max_chunk_cnt]} %{[zeek_cols][endpoint_url]} %{[zeek_cols][sec_channel_id]} %{[zeek_cols][sec_policy_uri_len]} %{[zeek_cols][sec_policy_uri]} %{[zeek_cols][snd_cert_len]} %{[zeek_cols][snd_cert]} %{[zeek_cols][rcv_cert_len]} %{[zeek_cols][rcv_cert]} %{[zeek_cols][seq_number]} %{[zeek_cols][request_id]} %{[zeek_cols][encoding_mask]} %{[zeek_cols][namespace_idx]} %{[zeek_cols][identifier]} %{[zeek_cols][identifier_str]} %{[zeek_cols][req_hdr_node_id_type]} %{[zeek_cols][req_hdr_node_id_namespace_idx]} %{[zeek_cols][req_hdr_node_id_numeric]} %{[zeek_cols][req_hdr_node_id_string]} %{[zeek_cols][req_hdr_node_id_guid]} %{[zeek_cols][req_hdr_node_id_opaque]} %{[zeek_cols][req_hdr_timestamp]} %{[zeek_cols][req_hdr_request_handle]} %{[zeek_cols][req_hdr_return_diag]} %{[zeek_cols][req_hdr_audit_entry_id]} %{[zeek_cols][req_hdr_timeout_hint]} %{[zeek_cols][req_hdr_add_hdr_type_id]} %{[zeek_cols][req_hdr_add_hdr_enc_mask]} %{[zeek_cols][res_hdr_timestamp]} %{[zeek_cols][res_hdr_request_handle]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][res_hdr_service_diag_encoding]} %{[zeek_cols][res_hdr_add_hdr_type_id]} %{[zeek_cols][res_hdr_add_hdr_enc_mask]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary" - init => "@zeek_opcua_binary_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'msg_type', 'is_final', 'msg_size', 'error', 'reason', 'version', 'rcv_buf_size', 'snd_buf_size', 'max_msg_size', 'max_chunk_cnt', 'endpoint_url', 'sec_channel_id', 'sec_policy_uri_len', 'sec_policy_uri', 'snd_cert_len', 'snd_cert', 'rcv_cert_len', 'rcv_cert', 'seq_number', 'request_id', 'encoding_mask', 'namespace_idx', 'identifier', 'identifier_str', 'req_hdr_node_id_type', 'req_hdr_node_id_namespace_idx', 'req_hdr_node_id_numeric', 'req_hdr_node_id_string', 'req_hdr_node_id_guid', 'req_hdr_node_id_opaque', 'req_hdr_timestamp', 'req_hdr_request_handle', 'req_hdr_return_diag', 'req_hdr_audit_entry_id', 'req_hdr_timeout_hint', 'req_hdr_add_hdr_type_id', 'req_hdr_add_hdr_enc_mask', 'res_hdr_timestamp', 'res_hdr_request_handle', 'status_code_link_id', 'res_hdr_service_diag_encoding', 'res_hdr_add_hdr_type_id', 'res_hdr_add_hdr_enc_mask' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary" + init => "@zeek_opcua_binary_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'msg_type', 'is_final', 'msg_size', 'error', 'reason', 'version', 'rcv_buf_size', 'snd_buf_size', 'max_msg_size', 'max_chunk_cnt', 'endpoint_url', 'sec_channel_id', 'sec_policy_uri_len', 'sec_policy_uri', 'snd_cert_len', 'snd_cert', 'rcv_cert_len', 'rcv_cert', 'seq_number', 'request_id', 'encoding_mask', 'namespace_idx', 'identifier', 'identifier_str', 'req_hdr_node_id_type', 'req_hdr_node_id_namespace_idx', 'req_hdr_node_id_numeric', 'req_hdr_node_id_string', 'req_hdr_node_id_guid', 'req_hdr_node_id_opaque', 'req_hdr_timestamp', 'req_hdr_request_handle', 'req_hdr_return_diag', 'req_hdr_audit_entry_id', 'req_hdr_timeout_hint', 'req_hdr_add_hdr_type_id', 'req_hdr_add_hdr_enc_mask', 'res_hdr_timestamp', 'res_hdr_request_handle', 'status_code_link_id', 'res_hdr_service_diag_encoding', 'res_hdr_add_hdr_type_id', 'res_hdr_add_hdr_enc_mask' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3845,24 +3839,27 @@ filter { # opcua_binary_activate_session.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_activate_session" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][client_algorithm]} %{[zeek_cols][client_signature]} %{[zeek_cols][client_software_cert_link_id]} %{[zeek_cols][opcua_locale_link_id]} %{[zeek_cols][ext_obj_type_id_encoding_mask]} %{[zeek_cols][ext_obj_type_id_namespace_idx]} %{[zeek_cols][ext_obj_type_id_numeric]} %{[zeek_cols][ext_obj_type_id_string]} %{[zeek_cols][ext_obj_type_id_guid]} %{[zeek_cols][ext_obj_type_id_opaque]} %{[zeek_cols][ext_obj_type_id_str]} %{[zeek_cols][ext_obj_encoding]} %{[zeek_cols][ext_obj_policy_id]} %{[zeek_cols][ext_obj_user_name]} %{[zeek_cols][ext_obj_password]} %{[zeek_cols][ext_obj_encryption_algorithom]} %{[zeek_cols][ext_obj_certificate_data]} %{[zeek_cols][ext_obj_token_data]} %{[zeek_cols][user_token_algorithm]} %{[zeek_cols][user_token_signature]} %{[zeek_cols][server_nonce]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][activate_session_diag_info_link_id]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_activate_session" + } else { + dissect { + id => "dissect_zeek_opcua_binary_activate_session" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][client_algorithm]} %{[zeek_cols][client_signature]} %{[zeek_cols][client_software_cert_link_id]} %{[zeek_cols][opcua_locale_link_id]} %{[zeek_cols][ext_obj_type_id_encoding_mask]} %{[zeek_cols][ext_obj_type_id_namespace_idx]} %{[zeek_cols][ext_obj_type_id_numeric]} %{[zeek_cols][ext_obj_type_id_string]} %{[zeek_cols][ext_obj_type_id_guid]} %{[zeek_cols][ext_obj_type_id_opaque]} %{[zeek_cols][ext_obj_type_id_str]} %{[zeek_cols][ext_obj_encoding]} %{[zeek_cols][ext_obj_policy_id]} %{[zeek_cols][ext_obj_user_name]} %{[zeek_cols][ext_obj_password]} %{[zeek_cols][ext_obj_encryption_algorithom]} %{[zeek_cols][ext_obj_certificate_data]} %{[zeek_cols][ext_obj_token_data]} %{[zeek_cols][user_token_algorithm]} %{[zeek_cols][user_token_signature]} %{[zeek_cols][server_nonce]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][activate_session_diag_info_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_activate_session" - init => "@zeek_opcua_binary_activate_session_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'client_algorithm', 'client_signature', 'client_software_cert_link_id', 'opcua_locale_link_id', 'ext_obj_type_id_encoding_mask', 'ext_obj_type_id_namespace_idx', 'ext_obj_type_id_numeric', 'ext_obj_type_id_string', 'ext_obj_type_id_guid', 'ext_obj_type_id_opaque', 'ext_obj_type_id_str', 'ext_obj_encoding', 'ext_obj_policy_id', 'ext_obj_user_name', 'ext_obj_password', 'ext_obj_encryption_algorithom', 'ext_obj_certificate_data', 'ext_obj_token_data', 'user_token_algorithm', 'user_token_signature', 'server_nonce', 'status_code_link_id', 'activate_session_diag_info_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_activate_session_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_activate_session" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_activate_session" + init => "@zeek_opcua_binary_activate_session_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'client_algorithm', 'client_signature', 'client_software_cert_link_id', 'opcua_locale_link_id', 'ext_obj_type_id_encoding_mask', 'ext_obj_type_id_namespace_idx', 'ext_obj_type_id_numeric', 'ext_obj_type_id_string', 'ext_obj_type_id_guid', 'ext_obj_type_id_opaque', 'ext_obj_type_id_str', 'ext_obj_encoding', 'ext_obj_policy_id', 'ext_obj_user_name', 'ext_obj_password', 'ext_obj_encryption_algorithom', 'ext_obj_certificate_data', 'ext_obj_token_data', 'user_token_algorithm', 'user_token_signature', 'server_nonce', 'status_code_link_id', 'activate_session_diag_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_activate_session_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3880,24 +3877,27 @@ filter { # opcua_binary_activate_session_client_software_cert.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_activate_session_client_software_cert" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][client_software_cert_link_id]} %{[zeek_cols][cert_data]} %{[zeek_cols][cert_signature]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_activate_session_client_software_cert" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_opcua_binary_activate_session_client_software_cert" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][client_software_cert_link_id]} %{[zeek_cols][cert_data]} %{[zeek_cols][cert_signature]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_activate_session_client_software_cert" - init => "@zeek_opcua_binary_activate_session_client_software_cert_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'client_software_cert_link_id', 'cert_data', 'cert_signature' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_activate_session_client_software_cert_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_activate_session_client_software_cert" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_activate_session_client_software_cert" + init => "@zeek_opcua_binary_activate_session_client_software_cert_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'client_software_cert_link_id', 'cert_data', 'cert_signature' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_activate_session_client_software_cert_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3915,24 +3915,27 @@ filter { # opcua_binary_activate_session_locale_id.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_activate_session_locale_id" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_locale_link_id]} %{[zeek_cols][local_id]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_activate_session_locale_id" + } else { + dissect { + id => "dissect_zeek_opcua_binary_activate_session_locale_id" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_locale_link_id]} %{[zeek_cols][local_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_activate_session_locale_id" - init => "@zeek_opcua_binary_activate_session_locale_id_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_locale_link_id', 'local_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_activate_session_locale_id_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_activate_session_locale_id" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_activate_session_locale_id" + init => "@zeek_opcua_binary_activate_session_locale_id_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_locale_link_id', 'local_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_activate_session_locale_id_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3950,24 +3953,27 @@ filter { # opcua_binary_aggregate_filter.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_aggregate_filter" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][monitored_parameters_link_id]} %{[zeek_cols][start_time]} %{[zeek_cols][start_time_str]} %{[zeek_cols][aggregate_type_encoding_mask]} %{[zeek_cols][aggregate_type_namespace_idx]} %{[zeek_cols][aggregate_type_numeric]} %{[zeek_cols][aggregate_type_string]} %{[zeek_cols][aggregate_type_guid]} %{[zeek_cols][aggregate_type_opaque]} %{[zeek_cols][processing_interval]} %{[zeek_cols][use_server_capabilities_default]} %{[zeek_cols][treat_uncertain_as_bad]} %{[zeek_cols][percent_data_good]} %{[zeek_cols][percent_data_bad]} %{[zeek_cols][use_slopped_extrapolation]} %{[zeek_cols][revised_start_time]} %{[zeek_cols][revised_start_time_str]} %{[zeek_cols][revised_processing_interval]} %{[zeek_cols][revised_use_server_capabilities_default]} %{[zeek_cols][revised_treat_uncertain_as_bad]} %{[zeek_cols][revised_percent_data_good]} %{[zeek_cols][revised_percent_data_bad]} %{[zeek_cols][revised_use_slopped_extrapolation]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_aggregate_filter" + } else { + dissect { + id => "dissect_zeek_opcua_binary_aggregate_filter" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][monitored_parameters_link_id]} %{[zeek_cols][start_time]} %{[zeek_cols][start_time_str]} %{[zeek_cols][aggregate_type_encoding_mask]} %{[zeek_cols][aggregate_type_namespace_idx]} %{[zeek_cols][aggregate_type_numeric]} %{[zeek_cols][aggregate_type_string]} %{[zeek_cols][aggregate_type_guid]} %{[zeek_cols][aggregate_type_opaque]} %{[zeek_cols][processing_interval]} %{[zeek_cols][use_server_capabilities_default]} %{[zeek_cols][treat_uncertain_as_bad]} %{[zeek_cols][percent_data_good]} %{[zeek_cols][percent_data_bad]} %{[zeek_cols][use_slopped_extrapolation]} %{[zeek_cols][revised_start_time]} %{[zeek_cols][revised_start_time_str]} %{[zeek_cols][revised_processing_interval]} %{[zeek_cols][revised_use_server_capabilities_default]} %{[zeek_cols][revised_treat_uncertain_as_bad]} %{[zeek_cols][revised_percent_data_good]} %{[zeek_cols][revised_percent_data_bad]} %{[zeek_cols][revised_use_slopped_extrapolation]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_aggregate_filter" - init => "@zeek_opcua_binary_aggregate_filter_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'monitored_parameters_link_id', 'start_time', 'start_time_str', 'aggregate_type_encoding_mask', 'aggregate_type_namespace_idx', 'aggregate_type_numeric', 'aggregate_type_string', 'aggregate_type_guid', 'aggregate_type_opaque', 'processing_interval', 'use_server_capabilities_default', 'treat_uncertain_as_bad', 'percent_data_good', 'percent_data_bad', 'use_slopped_extrapolation', 'revised_start_time', 'revised_start_time_str', 'revised_processing_interval', 'revised_use_server_capabilities_default', 'revised_treat_uncertain_as_bad', 'revised_percent_data_good', 'revised_percent_data_bad', 'revised_use_slopped_extrapolation' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_aggregate_filter_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_aggregate_filter" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_aggregate_filter" + init => "@zeek_opcua_binary_aggregate_filter_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'monitored_parameters_link_id', 'start_time', 'start_time_str', 'aggregate_type_encoding_mask', 'aggregate_type_namespace_idx', 'aggregate_type_numeric', 'aggregate_type_string', 'aggregate_type_guid', 'aggregate_type_opaque', 'processing_interval', 'use_server_capabilities_default', 'treat_uncertain_as_bad', 'percent_data_good', 'percent_data_bad', 'use_slopped_extrapolation', 'revised_start_time', 'revised_start_time_str', 'revised_processing_interval', 'revised_use_server_capabilities_default', 'revised_treat_uncertain_as_bad', 'revised_percent_data_good', 'revised_percent_data_bad', 'revised_use_slopped_extrapolation' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_aggregate_filter_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3985,24 +3991,27 @@ filter { # opcua_binary_event_filter_attribute_operand.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter_attribute_operand" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][node_id_encoding_mask]} %{[zeek_cols][node_id_namespace_idx]} %{[zeek_cols][node_id_numeric]} %{[zeek_cols][node_id_string]} %{[zeek_cols][node_id_guid]} %{[zeek_cols][node_id_opaque]} %{[zeek_cols][alias]} %{[zeek_cols][browse_path_element_link_id]} %{[zeek_cols][attribute]} %{[zeek_cols][index_range]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter_attribute_operand" + } else { + dissect { + id => "dissect_zeek_opcua_binary_event_filter_attribute_operand" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][node_id_encoding_mask]} %{[zeek_cols][node_id_namespace_idx]} %{[zeek_cols][node_id_numeric]} %{[zeek_cols][node_id_string]} %{[zeek_cols][node_id_guid]} %{[zeek_cols][node_id_opaque]} %{[zeek_cols][alias]} %{[zeek_cols][browse_path_element_link_id]} %{[zeek_cols][attribute]} %{[zeek_cols][index_range]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter_attribute_operand" - init => "@zeek_opcua_binary_event_filter_attribute_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'node_id_encoding_mask', 'node_id_namespace_idx', 'node_id_numeric', 'node_id_string', 'node_id_guid', 'node_id_opaque', 'alias', 'browse_path_element_link_id', 'attribute', 'index_range' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_attribute_operand_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter_attribute_operand" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter_attribute_operand" + init => "@zeek_opcua_binary_event_filter_attribute_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'node_id_encoding_mask', 'node_id_namespace_idx', 'node_id_numeric', 'node_id_string', 'node_id_guid', 'node_id_opaque', 'alias', 'browse_path_element_link_id', 'attribute', 'index_range' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_attribute_operand_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4020,24 +4029,27 @@ filter { # opcua_binary_event_filter_attribute_operand_browse_paths.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter_attribute_operand_browse_paths" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_path_element_link_id]} %{[zeek_cols][type_id_encoding_mask]} %{[zeek_cols][type_id_namespace_idx]} %{[zeek_cols][type_id_numeric]} %{[zeek_cols][type_id_string]} %{[zeek_cols][type_id_guid]} %{[zeek_cols][type_id_opaque]} %{[zeek_cols][is_inverse]} %{[zeek_cols][include_subtypes]} %{[zeek_cols][target_name_namespace_idx]} %{[zeek_cols][target_name]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter_attribute_operand_browse_paths" + } else { + dissect { + id => "dissect_zeek_opcua_binary_event_filter_attribute_operand_browse_paths" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_path_element_link_id]} %{[zeek_cols][type_id_encoding_mask]} %{[zeek_cols][type_id_namespace_idx]} %{[zeek_cols][type_id_numeric]} %{[zeek_cols][type_id_string]} %{[zeek_cols][type_id_guid]} %{[zeek_cols][type_id_opaque]} %{[zeek_cols][is_inverse]} %{[zeek_cols][include_subtypes]} %{[zeek_cols][target_name_namespace_idx]} %{[zeek_cols][target_name]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter_attribute_operand_browse_paths" - init => "@zeek_opcua_binary_event_filter_attribute_operand_browse_paths_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_path_element_link_id', 'type_id_encoding_mask', 'type_id_namespace_idx', 'type_id_numeric', 'type_id_string', 'type_id_guid', 'type_id_opaque', 'is_inverse', 'include_subtypes', 'target_name_namespace_idx', 'target_name' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_attribute_operand_browse_paths_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter_attribute_operand_browse_paths" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter_attribute_operand_browse_paths" + init => "@zeek_opcua_binary_event_filter_attribute_operand_browse_paths_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_path_element_link_id', 'type_id_encoding_mask', 'type_id_namespace_idx', 'type_id_numeric', 'type_id_string', 'type_id_guid', 'type_id_opaque', 'is_inverse', 'include_subtypes', 'target_name_namespace_idx', 'target_name' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_attribute_operand_browse_paths_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4055,24 +4067,27 @@ filter { # opcua_binary_browse.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_browse" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][browse_service_type]} %{[zeek_cols][browse_view_id_encoding_mask]} %{[zeek_cols][browse_view_id_namespace_idx]} %{[zeek_cols][browse_view_id_numeric]} %{[zeek_cols][browse_view_id_string]} %{[zeek_cols][browse_view_id_guid]} %{[zeek_cols][browse_view_id_opaque]} %{[zeek_cols][browse_view_description_timestamp]} %{[zeek_cols][browse_view_description_view_version]} %{[zeek_cols][req_max_ref_nodes]} %{[zeek_cols][browse_description_link_id]} %{[zeek_cols][browse_next_release_continuation_point]} %{[zeek_cols][browse_next_link_id]} %{[zeek_cols][browse_response_link_id]} %{[zeek_cols][browse_diag_info_link_id]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_browse" + } else { + dissect { + id => "dissect_zeek_opcua_binary_browse" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][browse_service_type]} %{[zeek_cols][browse_view_id_encoding_mask]} %{[zeek_cols][browse_view_id_namespace_idx]} %{[zeek_cols][browse_view_id_numeric]} %{[zeek_cols][browse_view_id_string]} %{[zeek_cols][browse_view_id_guid]} %{[zeek_cols][browse_view_id_opaque]} %{[zeek_cols][browse_view_description_timestamp]} %{[zeek_cols][browse_view_description_view_version]} %{[zeek_cols][req_max_ref_nodes]} %{[zeek_cols][browse_description_link_id]} %{[zeek_cols][browse_next_release_continuation_point]} %{[zeek_cols][browse_next_link_id]} %{[zeek_cols][browse_response_link_id]} %{[zeek_cols][browse_diag_info_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_browse" - init => "@zeek_opcua_binary_browse_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'browse_service_type', 'browse_view_id_encoding_mask', 'browse_view_id_namespace_idx', 'browse_view_id_numeric', 'browse_view_id_string', 'browse_view_id_guid', 'browse_view_id_opaque', 'browse_view_description_timestamp', 'browse_view_description_view_version', 'req_max_ref_nodes', 'browse_description_link_id', 'browse_next_release_continuation_point', 'browse_next_link_id', 'browse_response_link_id', 'browse_diag_info_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_browse" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_browse" + init => "@zeek_opcua_binary_browse_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'browse_service_type', 'browse_view_id_encoding_mask', 'browse_view_id_namespace_idx', 'browse_view_id_numeric', 'browse_view_id_string', 'browse_view_id_guid', 'browse_view_id_opaque', 'browse_view_description_timestamp', 'browse_view_description_view_version', 'req_max_ref_nodes', 'browse_description_link_id', 'browse_next_release_continuation_point', 'browse_next_link_id', 'browse_response_link_id', 'browse_diag_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4090,24 +4105,27 @@ filter { # opcua_binary_browse_description.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_browse_description" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_description_link_id]} %{[zeek_cols][browse_description_encoding_mask]} %{[zeek_cols][browse_description_namespace_idx]} %{[zeek_cols][browse_description_numeric]} %{[zeek_cols][browse_description_string]} %{[zeek_cols][browse_description_guid]} %{[zeek_cols][browse_description_opaque]} %{[zeek_cols][browse_direction]} %{[zeek_cols][browse_description_ref_encoding_mask]} %{[zeek_cols][browse_description_ref_namespace_idx]} %{[zeek_cols][browse_description_ref_numeric]} %{[zeek_cols][browse_description_ref_string]} %{[zeek_cols][browse_description_ref_guid]} %{[zeek_cols][browse_description_ref_opaque]} %{[zeek_cols][browse_description_include_subtypes]} %{[zeek_cols][browse_node_class_mask]} %{[zeek_cols][browse_result_mask]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_browse_description" + } else { + dissect { + id => "dissect_zeek_opcua_binary_browse_description" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_description_link_id]} %{[zeek_cols][browse_description_encoding_mask]} %{[zeek_cols][browse_description_namespace_idx]} %{[zeek_cols][browse_description_numeric]} %{[zeek_cols][browse_description_string]} %{[zeek_cols][browse_description_guid]} %{[zeek_cols][browse_description_opaque]} %{[zeek_cols][browse_direction]} %{[zeek_cols][browse_description_ref_encoding_mask]} %{[zeek_cols][browse_description_ref_namespace_idx]} %{[zeek_cols][browse_description_ref_numeric]} %{[zeek_cols][browse_description_ref_string]} %{[zeek_cols][browse_description_ref_guid]} %{[zeek_cols][browse_description_ref_opaque]} %{[zeek_cols][browse_description_include_subtypes]} %{[zeek_cols][browse_node_class_mask]} %{[zeek_cols][browse_result_mask]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_browse_description" - init => "@zeek_opcua_binary_browse_description_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_description_link_id', 'browse_description_encoding_mask', 'browse_description_namespace_idx', 'browse_description_numeric', 'browse_description_string', 'browse_description_guid', 'browse_description_opaque', 'browse_direction', 'browse_description_ref_encoding_mask', 'browse_description_ref_namespace_idx', 'browse_description_ref_numeric', 'browse_description_ref_string', 'browse_description_ref_guid', 'browse_description_ref_opaque', 'browse_description_include_subtypes', 'browse_node_class_mask', 'browse_result_mask' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_description_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_browse_description" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_browse_description" + init => "@zeek_opcua_binary_browse_description_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_description_link_id', 'browse_description_encoding_mask', 'browse_description_namespace_idx', 'browse_description_numeric', 'browse_description_string', 'browse_description_guid', 'browse_description_opaque', 'browse_direction', 'browse_description_ref_encoding_mask', 'browse_description_ref_namespace_idx', 'browse_description_ref_numeric', 'browse_description_ref_string', 'browse_description_ref_guid', 'browse_description_ref_opaque', 'browse_description_include_subtypes', 'browse_node_class_mask', 'browse_result_mask' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_description_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4125,24 +4143,27 @@ filter { # opcua_binary_browse_response_references.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_browse_response_references" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_reference_link_id]} %{[zeek_cols][browse_response_ref_encoding_mask]} %{[zeek_cols][browse_response_ref_namespace_idx]} %{[zeek_cols][browse_response_ref_numeric]} %{[zeek_cols][browse_response_ref_string]} %{[zeek_cols][browse_response_ref_guid]} %{[zeek_cols][browse_response_ref_opaque]} %{[zeek_cols][browse_response_is_forward]} %{[zeek_cols][browse_response_ref_type_encoding_mask]} %{[zeek_cols][browse_response_ref_type_namespace_idx]} %{[zeek_cols][browse_response_ref_type_numeric]} %{[zeek_cols][browse_response_ref_type_string]} %{[zeek_cols][browse_response_ref_type_guid]} %{[zeek_cols][browse_response_ref_type_opaque]} %{[zeek_cols][browse_response_ref_type_namespace_uri]} %{[zeek_cols][browse_response_ref_type_server_idx]} %{[zeek_cols][browse_response_ref_name_idx]} %{[zeek_cols][browse_response_ref_name]} %{[zeek_cols][browse_response_display_name_mask]} %{[zeek_cols][browse_response_display_name_locale]} %{[zeek_cols][browse_response_display_name_text]} %{[zeek_cols][browse_response_node_class]} %{[zeek_cols][browse_response_type_def_encoding_mask]} %{[zeek_cols][browse_response_type_def_namespace_idx]} %{[zeek_cols][browse_response_type_def_numeric]} %{[zeek_cols][browse_response_type_def_string]} %{[zeek_cols][browse_response_type_def_guid]} %{[zeek_cols][browse_response_type_def_opaque]} %{[zeek_cols][browse_response_type_def_namespace_uri]} %{[zeek_cols][browse_response_type_def_server_idx]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_browse_response_references" + } else { + dissect { + id => "dissect_zeek_opcua_binary_browse_response_references" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_reference_link_id]} %{[zeek_cols][browse_response_ref_encoding_mask]} %{[zeek_cols][browse_response_ref_namespace_idx]} %{[zeek_cols][browse_response_ref_numeric]} %{[zeek_cols][browse_response_ref_string]} %{[zeek_cols][browse_response_ref_guid]} %{[zeek_cols][browse_response_ref_opaque]} %{[zeek_cols][browse_response_is_forward]} %{[zeek_cols][browse_response_ref_type_encoding_mask]} %{[zeek_cols][browse_response_ref_type_namespace_idx]} %{[zeek_cols][browse_response_ref_type_numeric]} %{[zeek_cols][browse_response_ref_type_string]} %{[zeek_cols][browse_response_ref_type_guid]} %{[zeek_cols][browse_response_ref_type_opaque]} %{[zeek_cols][browse_response_ref_type_namespace_uri]} %{[zeek_cols][browse_response_ref_type_server_idx]} %{[zeek_cols][browse_response_ref_name_idx]} %{[zeek_cols][browse_response_ref_name]} %{[zeek_cols][browse_response_display_name_mask]} %{[zeek_cols][browse_response_display_name_locale]} %{[zeek_cols][browse_response_display_name_text]} %{[zeek_cols][browse_response_node_class]} %{[zeek_cols][browse_response_type_def_encoding_mask]} %{[zeek_cols][browse_response_type_def_namespace_idx]} %{[zeek_cols][browse_response_type_def_numeric]} %{[zeek_cols][browse_response_type_def_string]} %{[zeek_cols][browse_response_type_def_guid]} %{[zeek_cols][browse_response_type_def_opaque]} %{[zeek_cols][browse_response_type_def_namespace_uri]} %{[zeek_cols][browse_response_type_def_server_idx]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_browse_response_references" - init => "@zeek_opcua_binary_browse_response_references_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_reference_link_id', 'browse_response_ref_encoding_mask', 'browse_response_ref_namespace_idx', 'browse_response_ref_numeric', 'browse_response_ref_string', 'browse_response_ref_guid', 'browse_response_ref_opaque', 'browse_response_is_forward', 'browse_response_ref_type_encoding_mask', 'browse_response_ref_type_namespace_idx', 'browse_response_ref_type_numeric', 'browse_response_ref_type_string', 'browse_response_ref_type_guid', 'browse_response_ref_type_opaque', 'browse_response_ref_type_namespace_uri', 'browse_response_ref_type_server_idx', 'browse_response_ref_name_idx', 'browse_response_ref_name', 'browse_response_display_name_mask', 'browse_response_display_name_locale', 'browse_response_display_name_text', 'browse_response_node_class', 'browse_response_type_def_encoding_mask', 'browse_response_type_def_namespace_idx', 'browse_response_type_def_numeric', 'browse_response_type_def_string', 'browse_response_type_def_guid', 'browse_response_type_def_opaque', 'browse_response_type_def_namespace_uri', 'browse_response_type_def_server_idx' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_response_references_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_browse_response_references" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_browse_response_references" + init => "@zeek_opcua_binary_browse_response_references_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_reference_link_id', 'browse_response_ref_encoding_mask', 'browse_response_ref_namespace_idx', 'browse_response_ref_numeric', 'browse_response_ref_string', 'browse_response_ref_guid', 'browse_response_ref_opaque', 'browse_response_is_forward', 'browse_response_ref_type_encoding_mask', 'browse_response_ref_type_namespace_idx', 'browse_response_ref_type_numeric', 'browse_response_ref_type_string', 'browse_response_ref_type_guid', 'browse_response_ref_type_opaque', 'browse_response_ref_type_namespace_uri', 'browse_response_ref_type_server_idx', 'browse_response_ref_name_idx', 'browse_response_ref_name', 'browse_response_display_name_mask', 'browse_response_display_name_locale', 'browse_response_display_name_text', 'browse_response_node_class', 'browse_response_type_def_encoding_mask', 'browse_response_type_def_namespace_idx', 'browse_response_type_def_numeric', 'browse_response_type_def_string', 'browse_response_type_def_guid', 'browse_response_type_def_opaque', 'browse_response_type_def_namespace_uri', 'browse_response_type_def_server_idx' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_response_references_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4160,24 +4181,27 @@ filter { # opcua_binary_browse_request_continuation_point.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_browse_request_continuation_point" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_next_link_id]} %{[zeek_cols][continuation_point]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_browse_request_continuation_point" + } else { + dissect { + id => "dissect_zeek_opcua_binary_browse_request_continuation_point" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_next_link_id]} %{[zeek_cols][continuation_point]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_browse_request_continuation_point" - init => "@zeek_opcua_binary_browse_request_continuation_point_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_next_link_id', 'continuation_point' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_request_continuation_point_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_browse_request_continuation_point" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_browse_request_continuation_point" + init => "@zeek_opcua_binary_browse_request_continuation_point_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_next_link_id', 'continuation_point' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_request_continuation_point_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4195,24 +4219,27 @@ filter { # opcua_binary_browse_result.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_browse_result" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_response_link_id]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][browse_result_continuation_point]} %{[zeek_cols][browse_reference_link_id]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_browse_result" + } else { + dissect { + id => "dissect_zeek_opcua_binary_browse_result" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_response_link_id]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][browse_result_continuation_point]} %{[zeek_cols][browse_reference_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_browse_result" - init => "@zeek_opcua_binary_browse_result_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_response_link_id', 'status_code_link_id', 'browse_result_continuation_point', 'browse_reference_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_result_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_browse_result" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_browse_result" + init => "@zeek_opcua_binary_browse_result_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_response_link_id', 'status_code_link_id', 'browse_result_continuation_point', 'browse_reference_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_result_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4230,24 +4257,27 @@ filter { # opcua_binary_close_session.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_close_session" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][del_subscriptions]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_close_session" + } else { + dissect { + id => "dissect_zeek_opcua_binary_close_session" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][del_subscriptions]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_close_session" - init => "@zeek_opcua_binary_close_session_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'del_subscriptions' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_close_session_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_close_session" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_close_session" + init => "@zeek_opcua_binary_close_session_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'del_subscriptions' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_close_session_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4265,24 +4295,27 @@ filter { # opcua_binary_event_filter_where_clause.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter_where_clause" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][where_clause_link_id]} %{[zeek_cols][content_filter_element_link_id]} %{[zeek_cols][content_filter_status_code_link_id]} %{[zeek_cols][content_filter_diag_info_link_id]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter_where_clause" + } else { + dissect { + id => "dissect_zeek_opcua_binary_event_filter_where_clause" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][where_clause_link_id]} %{[zeek_cols][content_filter_element_link_id]} %{[zeek_cols][content_filter_status_code_link_id]} %{[zeek_cols][content_filter_diag_info_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter_where_clause" - init => "@zeek_opcua_binary_event_filter_where_clause_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'where_clause_link_id', 'content_filter_element_link_id', 'content_filter_status_code_link_id', 'content_filter_diag_info_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_where_clause_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter_where_clause" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter_where_clause" + init => "@zeek_opcua_binary_event_filter_where_clause_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'where_clause_link_id', 'content_filter_element_link_id', 'content_filter_status_code_link_id', 'content_filter_diag_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_where_clause_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4300,24 +4333,27 @@ filter { # opcua_binary_event_filter_where_clause_elements.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter_where_clause_elements" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_element_link_id]} %{[zeek_cols][filter_operator]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_encoding_mask]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_namespace_idx]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_numeric]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_string]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_guid]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_opaque]} %{[zeek_cols][content_filter_filter_operand_type_id_string]} %{[zeek_cols][content_filter_filter_operand_type_id_encoding]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][content_filter_operand_status_code_link_id]} %{[zeek_cols][content_filter_operand_diag_info_link_id]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter_where_clause_elements" + } else { + dissect { + id => "dissect_zeek_opcua_binary_event_filter_where_clause_elements" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_element_link_id]} %{[zeek_cols][filter_operator]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_encoding_mask]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_namespace_idx]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_numeric]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_string]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_guid]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_opaque]} %{[zeek_cols][content_filter_filter_operand_type_id_string]} %{[zeek_cols][content_filter_filter_operand_type_id_encoding]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][content_filter_operand_status_code_link_id]} %{[zeek_cols][content_filter_operand_diag_info_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter_where_clause_elements" - init => "@zeek_opcua_binary_event_filter_where_clause_elements_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_element_link_id', 'filter_operator', 'content_filter_filter_operand_type_id_node_id_encoding_mask', 'content_filter_filter_operand_type_id_node_id_namespace_idx', 'content_filter_filter_operand_type_id_node_id_numeric', 'content_filter_filter_operand_type_id_node_id_string', 'content_filter_filter_operand_type_id_node_id_guid', 'content_filter_filter_operand_type_id_node_id_opaque', 'content_filter_filter_operand_type_id_string', 'content_filter_filter_operand_type_id_encoding', 'content_filter_filter_operand_link_id', 'content_filter_operand_status_code_link_id', 'content_filter_operand_diag_info_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_where_clause_elements_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter_where_clause_elements" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter_where_clause_elements" + init => "@zeek_opcua_binary_event_filter_where_clause_elements_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_element_link_id', 'filter_operator', 'content_filter_filter_operand_type_id_node_id_encoding_mask', 'content_filter_filter_operand_type_id_node_id_namespace_idx', 'content_filter_filter_operand_type_id_node_id_numeric', 'content_filter_filter_operand_type_id_node_id_string', 'content_filter_filter_operand_type_id_node_id_guid', 'content_filter_filter_operand_type_id_node_id_opaque', 'content_filter_filter_operand_type_id_string', 'content_filter_filter_operand_type_id_encoding', 'content_filter_filter_operand_link_id', 'content_filter_operand_status_code_link_id', 'content_filter_operand_diag_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_where_clause_elements_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4335,24 +4371,27 @@ filter { # opcua_binary_create_monitored_items.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_create_monitored_items" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][subscription_id]} %{[zeek_cols][timestamps_to_return]} %{[zeek_cols][timestamps_to_return_str]} %{[zeek_cols][create_item_link_id]} %{[zeek_cols][create_monitored_items_diag_info_link_id]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_create_monitored_items" + } else { + dissect { + id => "dissect_zeek_opcua_binary_create_monitored_items" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][subscription_id]} %{[zeek_cols][timestamps_to_return]} %{[zeek_cols][timestamps_to_return_str]} %{[zeek_cols][create_item_link_id]} %{[zeek_cols][create_monitored_items_diag_info_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_create_monitored_items" - init => "@zeek_opcua_binary_create_monitored_items_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'subscription_id', 'timestamps_to_return', 'timestamps_to_return_str', 'create_item_link_id', 'create_monitored_items_diag_info_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_monitored_items_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_create_monitored_items" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_create_monitored_items" + init => "@zeek_opcua_binary_create_monitored_items_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'subscription_id', 'timestamps_to_return', 'timestamps_to_return_str', 'create_item_link_id', 'create_monitored_items_diag_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_monitored_items_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4370,24 +4409,27 @@ filter { # opcua_binary_create_monitored_items_create_item.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_create_monitored_items_create_item" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][create_item_link_id]} %{[zeek_cols][item_to_monitor_node_id_encoding_mask]} %{[zeek_cols][item_to_monitor_node_id_namespace_idx]} %{[zeek_cols][item_to_monitor_node_id_numeric]} %{[zeek_cols][item_to_monitor_node_id_string]} %{[zeek_cols][item_to_monitor_node_id_guid]} %{[zeek_cols][item_to_monitor_node_id_opaque]} %{[zeek_cols][item_to_monitor_attribute_id]} %{[zeek_cols][item_to_monitor_index_range]} %{[zeek_cols][item_to_monitor_namespace_idx]} %{[zeek_cols][item_to_monitor_name]} %{[zeek_cols][monitoring_mode]} %{[zeek_cols][monitoring_parameters_client_handle]} %{[zeek_cols][monitoring_parameters_sampling_interval]} %{[zeek_cols][monitoring_parameters_queue_size]} %{[zeek_cols][monitoring_parameters_discard_oldest]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_encoding_mask]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_namespace_idx]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_numeric]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_string]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_guid]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_opaque]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_string]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_encoding]} %{[zeek_cols][filter_info_details_link_id]} %{[zeek_cols][monitoring_parameters_status_code_link_id]} %{[zeek_cols][monitored_item_index_id]} %{[zeek_cols][monitoring_parameters_revised_sampling_interval]} %{[zeek_cols][monitoring_parameters_revised_queue_size]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_create_monitored_items_create_item" + } else { + dissect { + id => "dissect_zeek_opcua_binary_create_monitored_items_create_item" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][create_item_link_id]} %{[zeek_cols][item_to_monitor_node_id_encoding_mask]} %{[zeek_cols][item_to_monitor_node_id_namespace_idx]} %{[zeek_cols][item_to_monitor_node_id_numeric]} %{[zeek_cols][item_to_monitor_node_id_string]} %{[zeek_cols][item_to_monitor_node_id_guid]} %{[zeek_cols][item_to_monitor_node_id_opaque]} %{[zeek_cols][item_to_monitor_attribute_id]} %{[zeek_cols][item_to_monitor_index_range]} %{[zeek_cols][item_to_monitor_namespace_idx]} %{[zeek_cols][item_to_monitor_name]} %{[zeek_cols][monitoring_mode]} %{[zeek_cols][monitoring_parameters_client_handle]} %{[zeek_cols][monitoring_parameters_sampling_interval]} %{[zeek_cols][monitoring_parameters_queue_size]} %{[zeek_cols][monitoring_parameters_discard_oldest]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_encoding_mask]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_namespace_idx]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_numeric]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_string]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_guid]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_opaque]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_string]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_encoding]} %{[zeek_cols][filter_info_details_link_id]} %{[zeek_cols][monitoring_parameters_status_code_link_id]} %{[zeek_cols][monitored_item_index_id]} %{[zeek_cols][monitoring_parameters_revised_sampling_interval]} %{[zeek_cols][monitoring_parameters_revised_queue_size]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_create_monitored_items_create_item" - init => "@zeek_opcua_binary_create_monitored_items_create_item_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'create_item_link_id', 'item_to_monitor_node_id_encoding_mask', 'item_to_monitor_node_id_namespace_idx', 'item_to_monitor_node_id_numeric', 'item_to_monitor_node_id_string', 'item_to_monitor_node_id_guid', 'item_to_monitor_node_id_opaque', 'item_to_monitor_attribute_id', 'item_to_monitor_index_range', 'item_to_monitor_namespace_idx', 'item_to_monitor_name', 'monitoring_mode', 'monitoring_parameters_client_handle', 'monitoring_parameters_sampling_interval', 'monitoring_parameters_queue_size', 'monitoring_parameters_discard_oldest', 'monitoring_parameters_filter_info_type_id_node_id_encoding_mask', 'monitoring_parameters_filter_info_type_id_node_id_namespace_idx', 'monitoring_parameters_filter_info_type_id_node_id_numeric', 'monitoring_parameters_filter_info_type_id_node_id_string', 'monitoring_parameters_filter_info_type_id_node_id_guid', 'monitoring_parameters_filter_info_type_id_node_id_opaque', 'monitoring_parameters_filter_info_type_id_string', 'monitoring_parameters_filter_info_type_id_encoding', 'filter_info_details_link_id', 'monitoring_parameters_status_code_link_id', 'monitored_item_index_id', 'monitoring_parameters_revised_sampling_interval', 'monitoring_parameters_revised_queue_size' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_monitored_items_create_item_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_create_monitored_items_create_item" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_create_monitored_items_create_item" + init => "@zeek_opcua_binary_create_monitored_items_create_item_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'create_item_link_id', 'item_to_monitor_node_id_encoding_mask', 'item_to_monitor_node_id_namespace_idx', 'item_to_monitor_node_id_numeric', 'item_to_monitor_node_id_string', 'item_to_monitor_node_id_guid', 'item_to_monitor_node_id_opaque', 'item_to_monitor_attribute_id', 'item_to_monitor_index_range', 'item_to_monitor_namespace_idx', 'item_to_monitor_name', 'monitoring_mode', 'monitoring_parameters_client_handle', 'monitoring_parameters_sampling_interval', 'monitoring_parameters_queue_size', 'monitoring_parameters_discard_oldest', 'monitoring_parameters_filter_info_type_id_node_id_encoding_mask', 'monitoring_parameters_filter_info_type_id_node_id_namespace_idx', 'monitoring_parameters_filter_info_type_id_node_id_numeric', 'monitoring_parameters_filter_info_type_id_node_id_string', 'monitoring_parameters_filter_info_type_id_node_id_guid', 'monitoring_parameters_filter_info_type_id_node_id_opaque', 'monitoring_parameters_filter_info_type_id_string', 'monitoring_parameters_filter_info_type_id_encoding', 'filter_info_details_link_id', 'monitoring_parameters_status_code_link_id', 'monitored_item_index_id', 'monitoring_parameters_revised_sampling_interval', 'monitoring_parameters_revised_queue_size' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_monitored_items_create_item_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4405,24 +4447,27 @@ filter { # opcua_binary_create_session.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_create_session" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][application_uri]} %{[zeek_cols][product_uri]} %{[zeek_cols][encoding_mask]} %{[zeek_cols][locale]} %{[zeek_cols][text]} %{[zeek_cols][application_type]} %{[zeek_cols][gateway_server_uri]} %{[zeek_cols][discovery_profile_uri]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][server_uri]} %{[zeek_cols][endpoint_url]} %{[zeek_cols][session_name]} %{[zeek_cols][client_nonce]} %{[zeek_cols][client_cert_size]} %{[zeek_cols][client_cert]} %{[zeek_cols][req_session_timeout]} %{[zeek_cols][max_res_msg_size]} %{[zeek_cols][session_id_encoding_mask]} %{[zeek_cols][session_id_namespace_idx]} %{[zeek_cols][session_id_numeric]} %{[zeek_cols][session_id_string]} %{[zeek_cols][session_id_guid]} %{[zeek_cols][session_id_opaque]} %{[zeek_cols][auth_token_encoding_mask]} %{[zeek_cols][auth_token_namespace_idx]} %{[zeek_cols][auth_token_numeric]} %{[zeek_cols][auth_token_string]} %{[zeek_cols][auth_token_guid]} %{[zeek_cols][auth_token_opaque]} %{[zeek_cols][revised_session_timeout]} %{[zeek_cols][server_nonce]} %{[zeek_cols][server_cert_size]} %{[zeek_cols][server_cert]} %{[zeek_cols][endpoint_link_id]} %{[zeek_cols][algorithm]} %{[zeek_cols][signature]} %{[zeek_cols][max_req_msg_size]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_create_session" + } else { + dissect { + id => "dissect_zeek_opcua_binary_create_session" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][application_uri]} %{[zeek_cols][product_uri]} %{[zeek_cols][encoding_mask]} %{[zeek_cols][locale]} %{[zeek_cols][text]} %{[zeek_cols][application_type]} %{[zeek_cols][gateway_server_uri]} %{[zeek_cols][discovery_profile_uri]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][server_uri]} %{[zeek_cols][endpoint_url]} %{[zeek_cols][session_name]} %{[zeek_cols][client_nonce]} %{[zeek_cols][client_cert_size]} %{[zeek_cols][client_cert]} %{[zeek_cols][req_session_timeout]} %{[zeek_cols][max_res_msg_size]} %{[zeek_cols][session_id_encoding_mask]} %{[zeek_cols][session_id_namespace_idx]} %{[zeek_cols][session_id_numeric]} %{[zeek_cols][session_id_string]} %{[zeek_cols][session_id_guid]} %{[zeek_cols][session_id_opaque]} %{[zeek_cols][auth_token_encoding_mask]} %{[zeek_cols][auth_token_namespace_idx]} %{[zeek_cols][auth_token_numeric]} %{[zeek_cols][auth_token_string]} %{[zeek_cols][auth_token_guid]} %{[zeek_cols][auth_token_opaque]} %{[zeek_cols][revised_session_timeout]} %{[zeek_cols][server_nonce]} %{[zeek_cols][server_cert_size]} %{[zeek_cols][server_cert]} %{[zeek_cols][endpoint_link_id]} %{[zeek_cols][algorithm]} %{[zeek_cols][signature]} %{[zeek_cols][max_req_msg_size]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_create_session" - init => "@zeek_opcua_binary_create_session_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'application_uri', 'product_uri', 'encoding_mask', 'locale', 'text', 'application_type', 'gateway_server_uri', 'discovery_profile_uri', 'discovery_profile_link_id', 'server_uri', 'endpoint_url', 'session_name', 'client_nonce', 'client_cert_size', 'client_cert', 'req_session_timeout', 'max_res_msg_size', 'session_id_encoding_mask', 'session_id_namespace_idx', 'session_id_numeric', 'session_id_string', 'session_id_guid', 'session_id_opaque', 'auth_token_encoding_mask', 'auth_token_namespace_idx', 'auth_token_numeric', 'auth_token_string', 'auth_token_guid', 'auth_token_opaque', 'revised_session_timeout', 'server_nonce', 'server_cert_size', 'server_cert', 'endpoint_link_id', 'algorithm', 'signature', 'max_req_msg_size' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_create_session" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_create_session" + init => "@zeek_opcua_binary_create_session_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'application_uri', 'product_uri', 'encoding_mask', 'locale', 'text', 'application_type', 'gateway_server_uri', 'discovery_profile_uri', 'discovery_profile_link_id', 'server_uri', 'endpoint_url', 'session_name', 'client_nonce', 'client_cert_size', 'client_cert', 'req_session_timeout', 'max_res_msg_size', 'session_id_encoding_mask', 'session_id_namespace_idx', 'session_id_numeric', 'session_id_string', 'session_id_guid', 'session_id_opaque', 'auth_token_encoding_mask', 'auth_token_namespace_idx', 'auth_token_numeric', 'auth_token_string', 'auth_token_guid', 'auth_token_opaque', 'revised_session_timeout', 'server_nonce', 'server_cert_size', 'server_cert', 'endpoint_link_id', 'algorithm', 'signature', 'max_req_msg_size' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4440,24 +4485,27 @@ filter { # opcua_binary_create_session_discovery.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_create_session_discovery" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][discovery_profile_uri]} %{[zeek_cols][discovery_profile_url]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_create_session_discovery" + } else { + dissect { + id => "dissect_zeek_opcua_binary_create_session_discovery" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][discovery_profile_uri]} %{[zeek_cols][discovery_profile_url]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_create_session_discovery" - init => "@zeek_opcua_binary_create_session_discovery_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'discovery_profile_link_id', 'discovery_profile_uri', 'discovery_profile_url' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_discovery_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_create_session_discovery" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_create_session_discovery" + init => "@zeek_opcua_binary_create_session_discovery_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'discovery_profile_link_id', 'discovery_profile_uri', 'discovery_profile_url' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_discovery_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4475,24 +4523,27 @@ filter { # opcua_binary_create_session_endpoints.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_create_session_endpoints" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][endpoint_link_id]} %{[zeek_cols][endpoint_url]} %{[zeek_cols][application_uri]} %{[zeek_cols][product_uri]} %{[zeek_cols][encoding_mask]} %{[zeek_cols][locale]} %{[zeek_cols][text]} %{[zeek_cols][application_type]} %{[zeek_cols][gateway_server_uri]} %{[zeek_cols][discovery_profile_uri]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][cert_size]} %{[zeek_cols][server_cert]} %{[zeek_cols][message_security_mode]} %{[zeek_cols][security_policy_uri]} %{[zeek_cols][user_token_link_id]} %{[zeek_cols][transport_profile_uri]} %{[zeek_cols][security_level]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_create_session_endpoints" + } else { + dissect { + id => "dissect_zeek_opcua_binary_create_session_endpoints" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][endpoint_link_id]} %{[zeek_cols][endpoint_url]} %{[zeek_cols][application_uri]} %{[zeek_cols][product_uri]} %{[zeek_cols][encoding_mask]} %{[zeek_cols][locale]} %{[zeek_cols][text]} %{[zeek_cols][application_type]} %{[zeek_cols][gateway_server_uri]} %{[zeek_cols][discovery_profile_uri]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][cert_size]} %{[zeek_cols][server_cert]} %{[zeek_cols][message_security_mode]} %{[zeek_cols][security_policy_uri]} %{[zeek_cols][user_token_link_id]} %{[zeek_cols][transport_profile_uri]} %{[zeek_cols][security_level]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_create_session_endpoints" - init => "@zeek_opcua_binary_create_session_endpoints_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'endpoint_link_id', 'endpoint_url', 'application_uri', 'product_uri', 'encoding_mask', 'locale', 'text', 'application_type', 'gateway_server_uri', 'discovery_profile_uri', 'discovery_profile_link_id', 'cert_size', 'server_cert', 'message_security_mode', 'security_policy_uri', 'user_token_link_id', 'transport_profile_uri', 'security_level' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_endpoints_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_create_session_endpoints" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_create_session_endpoints" + init => "@zeek_opcua_binary_create_session_endpoints_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'endpoint_link_id', 'endpoint_url', 'application_uri', 'product_uri', 'encoding_mask', 'locale', 'text', 'application_type', 'gateway_server_uri', 'discovery_profile_uri', 'discovery_profile_link_id', 'cert_size', 'server_cert', 'message_security_mode', 'security_policy_uri', 'user_token_link_id', 'transport_profile_uri', 'security_level' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_endpoints_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4510,24 +4561,27 @@ filter { # opcua_binary_create_session_user_token.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_create_session_user_token" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user_token_link_id]} %{[zeek_cols][user_token_policy_id]} %{[zeek_cols][user_token_type]} %{[zeek_cols][user_token_issued_type]} %{[zeek_cols][user_token_endpoint_url]} %{[zeek_cols][user_token_sec_policy_uri]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_create_session_user_token" + } else { + dissect { + id => "dissect_zeek_opcua_binary_create_session_user_token" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user_token_link_id]} %{[zeek_cols][user_token_policy_id]} %{[zeek_cols][user_token_type]} %{[zeek_cols][user_token_issued_type]} %{[zeek_cols][user_token_endpoint_url]} %{[zeek_cols][user_token_sec_policy_uri]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_create_session_user_token" - init => "@zeek_opcua_binary_create_session_user_token_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'user_token_link_id', 'user_token_policy_id', 'user_token_type', 'user_token_issued_type', 'user_token_endpoint_url', 'user_token_sec_policy_uri' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_user_token_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_create_session_user_token" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_create_session_user_token" + init => "@zeek_opcua_binary_create_session_user_token_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'user_token_link_id', 'user_token_policy_id', 'user_token_type', 'user_token_issued_type', 'user_token_endpoint_url', 'user_token_sec_policy_uri' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_user_token_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4545,24 +4599,27 @@ filter { # opcua_binary_create_subscription.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_create_subscription" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][requested_publishing_interval]} %{[zeek_cols][requested_lifetime_count]} %{[zeek_cols][requested_max_keep_alive_count]} %{[zeek_cols][max_notifications_per_publish]} %{[zeek_cols][publishing_enabled]} %{[zeek_cols][priority]} %{[zeek_cols][subscription_id]} %{[zeek_cols][revised_publishing_interval]} %{[zeek_cols][revised_lifetime_count]} %{[zeek_cols][revised_max_keep_alive_count]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_create_subscription" + } else { + dissect { + id => "dissect_zeek_opcua_binary_create_subscription" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][requested_publishing_interval]} %{[zeek_cols][requested_lifetime_count]} %{[zeek_cols][requested_max_keep_alive_count]} %{[zeek_cols][max_notifications_per_publish]} %{[zeek_cols][publishing_enabled]} %{[zeek_cols][priority]} %{[zeek_cols][subscription_id]} %{[zeek_cols][revised_publishing_interval]} %{[zeek_cols][revised_lifetime_count]} %{[zeek_cols][revised_max_keep_alive_count]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_create_subscription" - init => "@zeek_opcua_binary_create_subscription_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'requested_publishing_interval', 'requested_lifetime_count', 'requested_max_keep_alive_count', 'max_notifications_per_publish', 'publishing_enabled', 'priority', 'subscription_id', 'revised_publishing_interval', 'revised_lifetime_count', 'revised_max_keep_alive_count' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_subscription_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_create_subscription" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_create_subscription" + init => "@zeek_opcua_binary_create_subscription_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'requested_publishing_interval', 'requested_lifetime_count', 'requested_max_keep_alive_count', 'max_notifications_per_publish', 'publishing_enabled', 'priority', 'subscription_id', 'revised_publishing_interval', 'revised_lifetime_count', 'revised_max_keep_alive_count' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_subscription_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4580,24 +4637,27 @@ filter { # opcua_binary_data_change_filter.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_data_change_filter" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][monitored_parameters_link_id]} %{[zeek_cols][trigger]} %{[zeek_cols][deadband_type]} %{[zeek_cols][deadband_value]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_data_change_filter" + } else { + dissect { + id => "dissect_zeek_opcua_binary_data_change_filter" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][monitored_parameters_link_id]} %{[zeek_cols][trigger]} %{[zeek_cols][deadband_type]} %{[zeek_cols][deadband_value]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_data_change_filter" - init => "@zeek_opcua_binary_data_change_filter_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'monitored_parameters_link_id', 'trigger', 'deadband_type', 'deadband_value' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_data_change_filter_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_data_change_filter" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_data_change_filter" + init => "@zeek_opcua_binary_data_change_filter_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'monitored_parameters_link_id', 'trigger', 'deadband_type', 'deadband_value' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_data_change_filter_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4615,24 +4675,27 @@ filter { # opcua_binary_diag_info_detail.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_diag_info_detail" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][diag_info_link_id]} %{[zeek_cols][root_object_id]} %{[zeek_cols][source]} %{[zeek_cols][source_str]} %{[zeek_cols][inner_diag_level]} %{[zeek_cols][has_symbolic_id]} %{[zeek_cols][symbolic_id]} %{[zeek_cols][symbolic_id_str]} %{[zeek_cols][has_namespace_uri]} %{[zeek_cols][namespace_uri]} %{[zeek_cols][namespace_uri_str]} %{[zeek_cols][has_locale]} %{[zeek_cols][locale]} %{[zeek_cols][locale_str]} %{[zeek_cols][has_locale_txt]} %{[zeek_cols][locale_txt]} %{[zeek_cols][locale_txt_str]} %{[zeek_cols][has_addl_info]} %{[zeek_cols][addl_info]} %{[zeek_cols][has_inner_stat_code]} %{[zeek_cols][inner_stat_code]} %{[zeek_cols][has_inner_diag_info]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_diag_info_detail" + } else { + dissect { + id => "dissect_zeek_opcua_binary_diag_info_detail" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][diag_info_link_id]} %{[zeek_cols][root_object_id]} %{[zeek_cols][source]} %{[zeek_cols][source_str]} %{[zeek_cols][inner_diag_level]} %{[zeek_cols][has_symbolic_id]} %{[zeek_cols][symbolic_id]} %{[zeek_cols][symbolic_id_str]} %{[zeek_cols][has_namespace_uri]} %{[zeek_cols][namespace_uri]} %{[zeek_cols][namespace_uri_str]} %{[zeek_cols][has_locale]} %{[zeek_cols][locale]} %{[zeek_cols][locale_str]} %{[zeek_cols][has_locale_txt]} %{[zeek_cols][locale_txt]} %{[zeek_cols][locale_txt_str]} %{[zeek_cols][has_addl_info]} %{[zeek_cols][addl_info]} %{[zeek_cols][has_inner_stat_code]} %{[zeek_cols][inner_stat_code]} %{[zeek_cols][has_inner_diag_info]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_diag_info_detail" - init => "@zeek_opcua_binary_diag_info_detail_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'diag_info_link_id', 'root_object_id', 'source', 'source_str', 'inner_diag_level', 'has_symbolic_id', 'symbolic_id', 'symbolic_id_str', 'has_namespace_uri', 'namespace_uri', 'namespace_uri_str', 'has_locale', 'locale', 'locale_str', 'has_locale_txt', 'locale_txt', 'locale_txt_str', 'has_addl_info', 'addl_info', 'has_inner_stat_code', 'inner_stat_code', 'has_inner_diag_info' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_diag_info_detail_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_diag_info_detail" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_diag_info_detail" + init => "@zeek_opcua_binary_diag_info_detail_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'diag_info_link_id', 'root_object_id', 'source', 'source_str', 'inner_diag_level', 'has_symbolic_id', 'symbolic_id', 'symbolic_id_str', 'has_namespace_uri', 'namespace_uri', 'namespace_uri_str', 'has_locale', 'locale', 'locale_str', 'has_locale_txt', 'locale_txt', 'locale_txt_str', 'has_addl_info', 'addl_info', 'has_inner_stat_code', 'inner_stat_code', 'has_inner_diag_info' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_diag_info_detail_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4650,24 +4713,27 @@ filter { # opcua_binary_event_filter_element_operand.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter_element_operand" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][element_index]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter_element_operand" + } else { + dissect { + id => "dissect_zeek_opcua_binary_event_filter_element_operand" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][element_index]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter_element_operand" - init => "@zeek_opcua_binary_event_filter_element_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'element_index' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_element_operand_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter_element_operand" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter_element_operand" + init => "@zeek_opcua_binary_event_filter_element_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'element_index' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_element_operand_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4685,24 +4751,27 @@ filter { # opcua_binary_event_filter.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][monitored_parameters_link_id]} %{[zeek_cols][select_clause_link_id]} %{[zeek_cols][where_clause_content_filter_link_id]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter" + } else { + dissect { + id => "dissect_zeek_opcua_binary_event_filter" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][monitored_parameters_link_id]} %{[zeek_cols][select_clause_link_id]} %{[zeek_cols][where_clause_content_filter_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter" - init => "@zeek_opcua_binary_event_filter_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'monitored_parameters_link_id', 'select_clause_link_id', 'where_clause_content_filter_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter" + init => "@zeek_opcua_binary_event_filter_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'monitored_parameters_link_id', 'select_clause_link_id', 'where_clause_content_filter_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4720,24 +4789,27 @@ filter { # opcua_binary_get_endpoints.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_get_endpoints" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][endpoint_url]} %{[zeek_cols][locale_link_id]} %{[zeek_cols][profile_uri_link_id]} %{[zeek_cols][endpoint_description_link_id]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_get_endpoints" + } else { + dissect { + id => "dissect_zeek_opcua_binary_get_endpoints" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][endpoint_url]} %{[zeek_cols][locale_link_id]} %{[zeek_cols][profile_uri_link_id]} %{[zeek_cols][endpoint_description_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_get_endpoints" - init => "@zeek_opcua_binary_get_endpoints_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'endpoint_url', 'locale_link_id', 'profile_uri_link_id', 'endpoint_description_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_get_endpoints" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_get_endpoints" + init => "@zeek_opcua_binary_get_endpoints_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'endpoint_url', 'locale_link_id', 'profile_uri_link_id', 'endpoint_description_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4755,24 +4827,27 @@ filter { # opcua_binary_get_endpoints_description.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_get_endpoints_description" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][endpoint_description_link_id]} %{[zeek_cols][endpoint_uri]} %{[zeek_cols][application_uri]} %{[zeek_cols][product_uri]} %{[zeek_cols][encoding_mask]} %{[zeek_cols][locale]} %{[zeek_cols][text]} %{[zeek_cols][application_type]} %{[zeek_cols][gateway_server_uri]} %{[zeek_cols][discovery_profile_uri]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][cert_size]} %{[zeek_cols][server_cert]} %{[zeek_cols][message_security_mode]} %{[zeek_cols][security_policy_uri]} %{[zeek_cols][user_token_link_id]} %{[zeek_cols][transport_profile_uri]} %{[zeek_cols][security_level]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_get_endpoints_description" + } else { + dissect { + id => "dissect_zeek_opcua_binary_get_endpoints_description" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][endpoint_description_link_id]} %{[zeek_cols][endpoint_uri]} %{[zeek_cols][application_uri]} %{[zeek_cols][product_uri]} %{[zeek_cols][encoding_mask]} %{[zeek_cols][locale]} %{[zeek_cols][text]} %{[zeek_cols][application_type]} %{[zeek_cols][gateway_server_uri]} %{[zeek_cols][discovery_profile_uri]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][cert_size]} %{[zeek_cols][server_cert]} %{[zeek_cols][message_security_mode]} %{[zeek_cols][security_policy_uri]} %{[zeek_cols][user_token_link_id]} %{[zeek_cols][transport_profile_uri]} %{[zeek_cols][security_level]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_get_endpoints_description" - init => "@zeek_opcua_binary_get_endpoints_description_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'endpoint_description_link_id', 'endpoint_uri', 'application_uri', 'product_uri', 'encoding_mask', 'locale', 'text', 'application_type', 'gateway_server_uri', 'discovery_profile_uri', 'discovery_profile_link_id', 'cert_size', 'server_cert', 'message_security_mode', 'security_policy_uri', 'user_token_link_id', 'transport_profile_uri', 'security_level' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_description_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_get_endpoints_description" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_get_endpoints_description" + init => "@zeek_opcua_binary_get_endpoints_description_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'endpoint_description_link_id', 'endpoint_uri', 'application_uri', 'product_uri', 'encoding_mask', 'locale', 'text', 'application_type', 'gateway_server_uri', 'discovery_profile_uri', 'discovery_profile_link_id', 'cert_size', 'server_cert', 'message_security_mode', 'security_policy_uri', 'user_token_link_id', 'transport_profile_uri', 'security_level' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_description_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4790,24 +4865,27 @@ filter { # opcua_binary_get_endpoints_discovery.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_get_endpoints_discovery" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][discovery_profile_url]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_get_endpoints_discovery" + } else { + dissect { + id => "dissect_zeek_opcua_binary_get_endpoints_discovery" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][discovery_profile_url]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_get_endpoints_discovery" - init => "@zeek_opcua_binary_get_endpoints_discovery_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'discovery_profile_link_id', 'discovery_profile_url' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_discovery_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_get_endpoints_discovery" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_get_endpoints_discovery" + init => "@zeek_opcua_binary_get_endpoints_discovery_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'discovery_profile_link_id', 'discovery_profile_url' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_discovery_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4825,24 +4903,27 @@ filter { # opcua_binary_get_endpoints_locale_id.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_get_endpoints_locale_id" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][locale_link_id]} %{[zeek_cols][locale_id]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_get_endpoints_locale_id" + } else { + dissect { + id => "dissect_zeek_opcua_binary_get_endpoints_locale_id" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][locale_link_id]} %{[zeek_cols][locale_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_get_endpoints_locale_id" - init => "@zeek_opcua_binary_get_endpoints_locale_id_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'locale_link_id', 'locale_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_locale_id_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_get_endpoints_locale_id" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_get_endpoints_locale_id" + init => "@zeek_opcua_binary_get_endpoints_locale_id_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'locale_link_id', 'locale_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_locale_id_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4860,24 +4941,27 @@ filter { # opcua_binary_get_endpoints_profile_uri.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_get_endpoints_profile_uri" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][profile_uri_link_id]} %{[zeek_cols][profile_uri]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_get_endpoints_profile_uri" + } else { + dissect { + id => "dissect_zeek_opcua_binary_get_endpoints_profile_uri" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][profile_uri_link_id]} %{[zeek_cols][profile_uri]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_get_endpoints_profile_uri" - init => "@zeek_opcua_binary_get_endpoints_profile_uri_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'profile_uri_link_id', 'profile_uri' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_profile_uri_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_get_endpoints_profile_uri" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_get_endpoints_profile_uri" + init => "@zeek_opcua_binary_get_endpoints_profile_uri_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'profile_uri_link_id', 'profile_uri' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_profile_uri_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4895,24 +4979,27 @@ filter { # opcua_binary_get_endpoints_user_token.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_get_endpoints_user_token" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user_token_link_id]} %{[zeek_cols][user_token_policy_id]} %{[zeek_cols][user_token_type]} %{[zeek_cols][user_token_issued_type]} %{[zeek_cols][user_token_endpoint_url]} %{[zeek_cols][user_token_sec_policy_uri]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_get_endpoints_user_token" + } else { + dissect { + id => "dissect_zeek_opcua_binary_get_endpoints_user_token" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user_token_link_id]} %{[zeek_cols][user_token_policy_id]} %{[zeek_cols][user_token_type]} %{[zeek_cols][user_token_issued_type]} %{[zeek_cols][user_token_endpoint_url]} %{[zeek_cols][user_token_sec_policy_uri]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_get_endpoints_user_token" - init => "@zeek_opcua_binary_get_endpoints_user_token_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'user_token_link_id', 'user_token_policy_id', 'user_token_type', 'user_token_issued_type', 'user_token_endpoint_url', 'user_token_sec_policy_uri' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_user_token_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_get_endpoints_user_token" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_get_endpoints_user_token" + init => "@zeek_opcua_binary_get_endpoints_user_token_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'user_token_link_id', 'user_token_policy_id', 'user_token_type', 'user_token_issued_type', 'user_token_endpoint_url', 'user_token_sec_policy_uri' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_user_token_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4930,24 +5017,27 @@ filter { # opcua_binary_event_filter_literal_operand.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter_literal_operand" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][literal_operand_variant_link]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter_literal_operand" + } else { + dissect { + id => "dissect_zeek_opcua_binary_event_filter_literal_operand" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][literal_operand_variant_link]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter_literal_operand" - init => "@zeek_opcua_binary_event_filter_literal_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'literal_operand_variant_link' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_literal_operand_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter_literal_operand" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter_literal_operand" + init => "@zeek_opcua_binary_event_filter_literal_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'literal_operand_variant_link' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_literal_operand_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4965,24 +5055,27 @@ filter { # opcua_binary_opensecure_channel.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_opensecure_channel" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][client_proto_ver]} %{[zeek_cols][sec_token_request_type]} %{[zeek_cols][message_security_mode]} %{[zeek_cols][client_nonce]} %{[zeek_cols][req_lifetime]} %{[zeek_cols][server_proto_ver]} %{[zeek_cols][sec_token_sec_channel_id]} %{[zeek_cols][sec_token_id]} %{[zeek_cols][sec_token_created_at]} %{[zeek_cols][sec_token_revised_time]} %{[zeek_cols][server_nonce]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_opensecure_channel" + } else { + dissect { + id => "dissect_zeek_opcua_binary_opensecure_channel" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][client_proto_ver]} %{[zeek_cols][sec_token_request_type]} %{[zeek_cols][message_security_mode]} %{[zeek_cols][client_nonce]} %{[zeek_cols][req_lifetime]} %{[zeek_cols][server_proto_ver]} %{[zeek_cols][sec_token_sec_channel_id]} %{[zeek_cols][sec_token_id]} %{[zeek_cols][sec_token_created_at]} %{[zeek_cols][sec_token_revised_time]} %{[zeek_cols][server_nonce]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_opensecure_channel" - init => "@zeek_opcua_binary_opensecure_channel_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'client_proto_ver', 'sec_token_request_type', 'message_security_mode', 'client_nonce', 'req_lifetime', 'server_proto_ver', 'sec_token_sec_channel_id', 'sec_token_id', 'sec_token_created_at', 'sec_token_revised_time', 'server_nonce' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_opensecure_channel_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_opensecure_channel" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_opensecure_channel" + init => "@zeek_opcua_binary_opensecure_channel_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'client_proto_ver', 'sec_token_request_type', 'message_security_mode', 'client_nonce', 'req_lifetime', 'server_proto_ver', 'sec_token_sec_channel_id', 'sec_token_id', 'sec_token_created_at', 'sec_token_revised_time', 'server_nonce' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_opensecure_channel_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5000,24 +5093,27 @@ filter { # opcua_binary_read.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_read" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][max_age]} %{[zeek_cols][timestamps_to_return]} %{[zeek_cols][timestamps_to_return_str]} %{[zeek_cols][nodes_to_read_link_id]} %{[zeek_cols][read_results_link_id]} %{[zeek_cols][diag_info_link_id]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_read" + } else { + dissect { + id => "dissect_zeek_opcua_binary_read" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][max_age]} %{[zeek_cols][timestamps_to_return]} %{[zeek_cols][timestamps_to_return_str]} %{[zeek_cols][nodes_to_read_link_id]} %{[zeek_cols][read_results_link_id]} %{[zeek_cols][diag_info_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_read" - init => "@zeek_opcua_binary_read_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'max_age', 'timestamps_to_return', 'timestamps_to_return_str', 'nodes_to_read_link_id', 'read_results_link_id', 'diag_info_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_read_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_read" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_read" + init => "@zeek_opcua_binary_read_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'max_age', 'timestamps_to_return', 'timestamps_to_return_str', 'nodes_to_read_link_id', 'read_results_link_id', 'diag_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_read_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5035,24 +5131,27 @@ filter { # opcua_binary_read_nodes_to_read.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_read_nodes_to_read" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][nodes_to_read_link_id]} %{[zeek_cols][node_id_encoding_mask]} %{[zeek_cols][node_id_namespace_idx]} %{[zeek_cols][node_id_numeric]} %{[zeek_cols][node_id_string]} %{[zeek_cols][node_id_guid]} %{[zeek_cols][node_id_opaque]} %{[zeek_cols][attribute_id]} %{[zeek_cols][attribute_id_str]} %{[zeek_cols][index_range]} %{[zeek_cols][data_encoding_name_idx]} %{[zeek_cols][data_encoding_name]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_read_nodes_to_read" + } else { + dissect { + id => "dissect_zeek_opcua_binary_read_nodes_to_read" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][nodes_to_read_link_id]} %{[zeek_cols][node_id_encoding_mask]} %{[zeek_cols][node_id_namespace_idx]} %{[zeek_cols][node_id_numeric]} %{[zeek_cols][node_id_string]} %{[zeek_cols][node_id_guid]} %{[zeek_cols][node_id_opaque]} %{[zeek_cols][attribute_id]} %{[zeek_cols][attribute_id_str]} %{[zeek_cols][index_range]} %{[zeek_cols][data_encoding_name_idx]} %{[zeek_cols][data_encoding_name]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_read_nodes_to_read" - init => "@zeek_opcua_binary_read_nodes_to_read_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'nodes_to_read_link_id', 'node_id_encoding_mask', 'node_id_namespace_idx', 'node_id_numeric', 'node_id_string', 'node_id_guid', 'node_id_opaque', 'attribute_id', 'attribute_id_str', 'index_range', 'data_encoding_name_idx', 'data_encoding_name' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_read_nodes_to_read_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_read_nodes_to_read" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_read_nodes_to_read" + init => "@zeek_opcua_binary_read_nodes_to_read_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'nodes_to_read_link_id', 'node_id_encoding_mask', 'node_id_namespace_idx', 'node_id_numeric', 'node_id_string', 'node_id_guid', 'node_id_opaque', 'attribute_id', 'attribute_id_str', 'index_range', 'data_encoding_name_idx', 'data_encoding_name' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_read_nodes_to_read_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5070,24 +5169,27 @@ filter { # opcua_binary_read_results.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_read_results" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][results_link_id]} %{[zeek_cols][level]} %{[zeek_cols][data_value_encoding_mask]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][source_timestamp]} %{[zeek_cols][source_pico_sec]} %{[zeek_cols][server_timestamp]} %{[zeek_cols][server_pico_sec]} %{[zeek_cols][read_results_variant_metadata_link_id]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_read_results" + } else { + dissect { + id => "dissect_zeek_opcua_binary_read_results" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][results_link_id]} %{[zeek_cols][level]} %{[zeek_cols][data_value_encoding_mask]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][source_timestamp]} %{[zeek_cols][source_pico_sec]} %{[zeek_cols][server_timestamp]} %{[zeek_cols][server_pico_sec]} %{[zeek_cols][read_results_variant_metadata_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_read_results" - init => "@zeek_opcua_binary_read_results_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'results_link_id', 'level', 'data_value_encoding_mask', 'status_code_link_id', 'source_timestamp', 'source_pico_sec', 'server_timestamp', 'server_pico_sec', 'read_results_variant_metadata_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_read_results_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_read_results" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_read_results" + init => "@zeek_opcua_binary_read_results_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'results_link_id', 'level', 'data_value_encoding_mask', 'status_code_link_id', 'source_timestamp', 'source_pico_sec', 'server_timestamp', 'server_pico_sec', 'read_results_variant_metadata_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_read_results_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5105,24 +5207,27 @@ filter { # opcua_binary_event_filter_select_clause.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter_select_clause" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][select_clause_link_id]} %{[zeek_cols][type_id_encoding_mask]} %{[zeek_cols][type_id_namespace_idx]} %{[zeek_cols][type_id_numeric]} %{[zeek_cols][type_id_string]} %{[zeek_cols][type_id_guid]} %{[zeek_cols][type_id_opaque]} %{[zeek_cols][simple_attribute_operand_browse_path_link_id]} %{[zeek_cols][attribute_id]} %{[zeek_cols][index_range]} %{[zeek_cols][select_clause_status_code_link_id]} %{[zeek_cols][select_clause_diagnostic_info_link_id]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter_select_clause" + } else { + dissect { + id => "dissect_zeek_opcua_binary_event_filter_select_clause" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][select_clause_link_id]} %{[zeek_cols][type_id_encoding_mask]} %{[zeek_cols][type_id_namespace_idx]} %{[zeek_cols][type_id_numeric]} %{[zeek_cols][type_id_string]} %{[zeek_cols][type_id_guid]} %{[zeek_cols][type_id_opaque]} %{[zeek_cols][simple_attribute_operand_browse_path_link_id]} %{[zeek_cols][attribute_id]} %{[zeek_cols][index_range]} %{[zeek_cols][select_clause_status_code_link_id]} %{[zeek_cols][select_clause_diagnostic_info_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter_select_clause" - init => "@zeek_opcua_binary_event_filter_select_clause_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'select_clause_link_id', 'type_id_encoding_mask', 'type_id_namespace_idx', 'type_id_numeric', 'type_id_string', 'type_id_guid', 'type_id_opaque', 'simple_attribute_operand_browse_path_link_id', 'attribute_id', 'index_range', 'select_clause_status_code_link_id', 'select_clause_diagnostic_info_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_select_clause_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter_select_clause" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter_select_clause" + init => "@zeek_opcua_binary_event_filter_select_clause_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'select_clause_link_id', 'type_id_encoding_mask', 'type_id_namespace_idx', 'type_id_numeric', 'type_id_string', 'type_id_guid', 'type_id_opaque', 'simple_attribute_operand_browse_path_link_id', 'attribute_id', 'index_range', 'select_clause_status_code_link_id', 'select_clause_diagnostic_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_select_clause_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5140,24 +5245,27 @@ filter { # opcua_binary_event_filter_simple_attribute_operand.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter_simple_attribute_operand" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][type_id_encoding_mask]} %{[zeek_cols][type_id_namespace_idx]} %{[zeek_cols][type_id_numeric]} %{[zeek_cols][type_id_string]} %{[zeek_cols][type_id_guid]} %{[zeek_cols][type_id_opaque]} %{[zeek_cols][simple_attribute_operand_browse_path_link_id]} %{[zeek_cols][attribute_id]} %{[zeek_cols][index_range]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter_simple_attribute_operand" + } else { + dissect { + id => "dissect_zeek_opcua_binary_event_filter_simple_attribute_operand" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][type_id_encoding_mask]} %{[zeek_cols][type_id_namespace_idx]} %{[zeek_cols][type_id_numeric]} %{[zeek_cols][type_id_string]} %{[zeek_cols][type_id_guid]} %{[zeek_cols][type_id_opaque]} %{[zeek_cols][simple_attribute_operand_browse_path_link_id]} %{[zeek_cols][attribute_id]} %{[zeek_cols][index_range]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter_simple_attribute_operand" - init => "@zeek_opcua_binary_event_filter_simple_attribute_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'type_id_encoding_mask', 'type_id_namespace_idx', 'type_id_numeric', 'type_id_string', 'type_id_guid', 'type_id_opaque', 'simple_attribute_operand_browse_path_link_id', 'attribute_id', 'index_range' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_simple_attribute_operand_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter_simple_attribute_operand" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter_simple_attribute_operand" + init => "@zeek_opcua_binary_event_filter_simple_attribute_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'type_id_encoding_mask', 'type_id_namespace_idx', 'type_id_numeric', 'type_id_string', 'type_id_guid', 'type_id_opaque', 'simple_attribute_operand_browse_path_link_id', 'attribute_id', 'index_range' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_simple_attribute_operand_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5175,24 +5283,27 @@ filter { # opcua_binary_event_filter_simple_attribute_operand_browse_paths.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][simple_attribute_operand_browse_path_link_id]} %{[zeek_cols][browse_path_src]} %{[zeek_cols][namespace_index]} %{[zeek_cols][name]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths" + } else { + dissect { + id => "dissect_zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][simple_attribute_operand_browse_path_link_id]} %{[zeek_cols][browse_path_src]} %{[zeek_cols][namespace_index]} %{[zeek_cols][name]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths" - init => "@zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'simple_attribute_operand_browse_path_link_id', 'browse_path_src', 'namespace_index', 'name' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths" + init => "@zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'simple_attribute_operand_browse_path_link_id', 'browse_path_src', 'namespace_index', 'name' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5210,24 +5321,27 @@ filter { # opcua_binary_status_code_detail.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_status_code_detail" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][source]} %{[zeek_cols][source_str]} %{[zeek_cols][source_level]} %{[zeek_cols][status_code]} %{[zeek_cols][severity]} %{[zeek_cols][severity_str]} %{[zeek_cols][sub_code]} %{[zeek_cols][sub_code_str]} %{[zeek_cols][structure_changed]} %{[zeek_cols][semantics_changed]} %{[zeek_cols][info_type]} %{[zeek_cols][info_type_str]} %{[zeek_cols][limit_bits]} %{[zeek_cols][limit_bits_str]} %{[zeek_cols][overflow]} %{[zeek_cols][historian_bits]} %{[zeek_cols][historian_bits_str]} %{[zeek_cols][historianpartial]} %{[zeek_cols][historianextradata]} %{[zeek_cols][historianmultivalue]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_status_code_detail" + } else { + dissect { + id => "dissect_zeek_opcua_binary_status_code_detail" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][source]} %{[zeek_cols][source_str]} %{[zeek_cols][source_level]} %{[zeek_cols][status_code]} %{[zeek_cols][severity]} %{[zeek_cols][severity_str]} %{[zeek_cols][sub_code]} %{[zeek_cols][sub_code_str]} %{[zeek_cols][structure_changed]} %{[zeek_cols][semantics_changed]} %{[zeek_cols][info_type]} %{[zeek_cols][info_type_str]} %{[zeek_cols][limit_bits]} %{[zeek_cols][limit_bits_str]} %{[zeek_cols][overflow]} %{[zeek_cols][historian_bits]} %{[zeek_cols][historian_bits_str]} %{[zeek_cols][historianpartial]} %{[zeek_cols][historianextradata]} %{[zeek_cols][historianmultivalue]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_status_code_detail" - init => "@zeek_opcua_binary_status_code_detail_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'status_code_link_id', 'source', 'source_str', 'source_level', 'status_code', 'severity', 'severity_str', 'sub_code', 'sub_code_str', 'structure_changed', 'semantics_changed', 'info_type', 'info_type_str', 'limit_bits', 'limit_bits_str', 'overflow', 'historian_bits', 'historian_bits_str', 'historianpartial', 'historianextradata', 'historianmultivalue' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_status_code_detail_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_status_code_detail" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_status_code_detail" + init => "@zeek_opcua_binary_status_code_detail_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'status_code_link_id', 'source', 'source_str', 'source_level', 'status_code', 'severity', 'severity_str', 'sub_code', 'sub_code_str', 'structure_changed', 'semantics_changed', 'info_type', 'info_type_str', 'limit_bits', 'limit_bits_str', 'overflow', 'historian_bits', 'historian_bits_str', 'historianpartial', 'historianextradata', 'historianmultivalue' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_status_code_detail_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5245,24 +5359,27 @@ filter { # opcua_binary_variant_array_dims.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_variant_array_dims" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][array_dim_link_id]} %{[zeek_cols][dimension]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_variant_array_dims" + } else { + dissect { + id => "dissect_zeek_opcua_binary_variant_array_dims" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][array_dim_link_id]} %{[zeek_cols][dimension]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_variant_array_dims" - init => "@zeek_opcua_binary_variant_array_dims_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'array_dim_link_id', 'dimension' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_array_dims_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_variant_array_dims" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_variant_array_dims" + init => "@zeek_opcua_binary_variant_array_dims_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'array_dim_link_id', 'dimension' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_array_dims_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5280,24 +5397,27 @@ filter { # opcua_binary_variant_data.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_variant_data" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][variant_data_link_id]} %{[zeek_cols][variant_data_value_signed_numeric]} %{[zeek_cols][variant_data_value_unsigned_numeric]} %{[zeek_cols][variant_data_value_string]} %{[zeek_cols][variant_data_node_id_encoding_mask]} %{[zeek_cols][variant_data_node_id_namespace_idx]} %{[zeek_cols][variant_data_node_id_numeric]} %{[zeek_cols][variant_data_node_id_string]} %{[zeek_cols][variant_data_node_id_guid]} %{[zeek_cols][variant_data_node_id_opaque]} %{[zeek_cols][variant_data_node_id_namespace_uri]} %{[zeek_cols][variant_data_node_id_server_idx]} %{[zeek_cols][variant_data_value_time]} %{[zeek_cols][variant_data_encoding_name_idx]} %{[zeek_cols][variant_data_encoding_name]} %{[zeek_cols][variant_data_mask]} %{[zeek_cols][variant_data_locale]} %{[zeek_cols][variant_data_text]} %{[zeek_cols][variant_data_value_decimal]} %{[zeek_cols][variant_data_status_code_link_id]} %{[zeek_cols][variant_data_diag_info_link_id]} %{[zeek_cols][variant_data_ext_obj_link_id]} %{[zeek_cols][variant_metadata_data_link_id]} %{[zeek_cols][variant_data_value_link_id]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_variant_data" + } else { + dissect { + id => "dissect_zeek_opcua_binary_variant_data" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][variant_data_link_id]} %{[zeek_cols][variant_data_value_signed_numeric]} %{[zeek_cols][variant_data_value_unsigned_numeric]} %{[zeek_cols][variant_data_value_string]} %{[zeek_cols][variant_data_node_id_encoding_mask]} %{[zeek_cols][variant_data_node_id_namespace_idx]} %{[zeek_cols][variant_data_node_id_numeric]} %{[zeek_cols][variant_data_node_id_string]} %{[zeek_cols][variant_data_node_id_guid]} %{[zeek_cols][variant_data_node_id_opaque]} %{[zeek_cols][variant_data_node_id_namespace_uri]} %{[zeek_cols][variant_data_node_id_server_idx]} %{[zeek_cols][variant_data_value_time]} %{[zeek_cols][variant_data_encoding_name_idx]} %{[zeek_cols][variant_data_encoding_name]} %{[zeek_cols][variant_data_mask]} %{[zeek_cols][variant_data_locale]} %{[zeek_cols][variant_data_text]} %{[zeek_cols][variant_data_value_decimal]} %{[zeek_cols][variant_data_status_code_link_id]} %{[zeek_cols][variant_data_diag_info_link_id]} %{[zeek_cols][variant_data_ext_obj_link_id]} %{[zeek_cols][variant_metadata_data_link_id]} %{[zeek_cols][variant_data_value_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_variant_data" - init => "@zeek_opcua_binary_variant_data_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'variant_data_link_id', 'variant_data_value_signed_numeric', 'variant_data_value_unsigned_numeric', 'variant_data_value_string', 'variant_data_node_id_encoding_mask', 'variant_data_node_id_namespace_idx', 'variant_data_node_id_numeric', 'variant_data_node_id_string', 'variant_data_node_id_guid', 'variant_data_node_id_opaque', 'variant_data_node_id_namespace_uri', 'variant_data_node_id_server_idx', 'variant_data_value_time', 'variant_data_encoding_name_idx', 'variant_data_encoding_name', 'variant_data_mask', 'variant_data_locale', 'variant_data_text', 'variant_data_value_decimal', 'variant_data_status_code_link_id', 'variant_data_diag_info_link_id', 'variant_data_ext_obj_link_id', 'variant_metadata_data_link_id', 'variant_data_value_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_data_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_variant_data" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_variant_data" + init => "@zeek_opcua_binary_variant_data_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'variant_data_link_id', 'variant_data_value_signed_numeric', 'variant_data_value_unsigned_numeric', 'variant_data_value_string', 'variant_data_node_id_encoding_mask', 'variant_data_node_id_namespace_idx', 'variant_data_node_id_numeric', 'variant_data_node_id_string', 'variant_data_node_id_guid', 'variant_data_node_id_opaque', 'variant_data_node_id_namespace_uri', 'variant_data_node_id_server_idx', 'variant_data_value_time', 'variant_data_encoding_name_idx', 'variant_data_encoding_name', 'variant_data_mask', 'variant_data_locale', 'variant_data_text', 'variant_data_value_decimal', 'variant_data_status_code_link_id', 'variant_data_diag_info_link_id', 'variant_data_ext_obj_link_id', 'variant_metadata_data_link_id', 'variant_data_value_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_data_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5315,24 +5435,27 @@ filter { # opcua_binary_variant_data_value.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_variant_data_value" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][variant_data_value_source_link]} %{[zeek_cols][data_value_encoding_mask]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][source_timestamp]} %{[zeek_cols][source_pico_sec]} %{[zeek_cols][server_timestamp]} %{[zeek_cols][server_pico_sec]} %{[zeek_cols][variant_metadata_link_id]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_variant_data_value" + } else { + dissect { + id => "dissect_zeek_opcua_binary_variant_data_value" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][variant_data_value_source_link]} %{[zeek_cols][data_value_encoding_mask]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][source_timestamp]} %{[zeek_cols][source_pico_sec]} %{[zeek_cols][server_timestamp]} %{[zeek_cols][server_pico_sec]} %{[zeek_cols][variant_metadata_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_variant_data_value" - init => "@zeek_opcua_binary_variant_data_value_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'variant_data_value_source_link', 'data_value_encoding_mask', 'status_code_link_id', 'source_timestamp', 'source_pico_sec', 'server_timestamp', 'server_pico_sec', 'variant_metadata_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_data_value_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_variant_data_value" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_variant_data_value" + init => "@zeek_opcua_binary_variant_data_value_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'variant_data_value_source_link', 'data_value_encoding_mask', 'status_code_link_id', 'source_timestamp', 'source_pico_sec', 'server_timestamp', 'server_pico_sec', 'variant_metadata_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_data_value_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5350,24 +5473,27 @@ filter { # opcua_binary_variant_extension_object.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_variant_extension_object" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][ext_obj_link_id]} %{[zeek_cols][ext_obj_node_id_encoding_mask]} %{[zeek_cols][ext_obj_node_id_namespace_idx]} %{[zeek_cols][ext_obj_node_id_numeric]} %{[zeek_cols][ext_obj_node_id_string]} %{[zeek_cols][ext_obj_node_id_guid]} %{[zeek_cols][ext_obj_node_id_opaque]} %{[zeek_cols][ext_obj_type_id_str]} %{[zeek_cols][ext_obj_encoding]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_variant_extension_object" + } else { + dissect { + id => "dissect_zeek_opcua_binary_variant_extension_object" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][ext_obj_link_id]} %{[zeek_cols][ext_obj_node_id_encoding_mask]} %{[zeek_cols][ext_obj_node_id_namespace_idx]} %{[zeek_cols][ext_obj_node_id_numeric]} %{[zeek_cols][ext_obj_node_id_string]} %{[zeek_cols][ext_obj_node_id_guid]} %{[zeek_cols][ext_obj_node_id_opaque]} %{[zeek_cols][ext_obj_type_id_str]} %{[zeek_cols][ext_obj_encoding]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_variant_extension_object" - init => "@zeek_opcua_binary_variant_extension_object_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'ext_obj_link_id', 'ext_obj_node_id_encoding_mask', 'ext_obj_node_id_namespace_idx', 'ext_obj_node_id_numeric', 'ext_obj_node_id_string', 'ext_obj_node_id_guid', 'ext_obj_node_id_opaque', 'ext_obj_type_id_str', 'ext_obj_encoding' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_extension_object_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_variant_extension_object" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_variant_extension_object" + init => "@zeek_opcua_binary_variant_extension_object_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'ext_obj_link_id', 'ext_obj_node_id_encoding_mask', 'ext_obj_node_id_namespace_idx', 'ext_obj_node_id_numeric', 'ext_obj_node_id_string', 'ext_obj_node_id_guid', 'ext_obj_node_id_opaque', 'ext_obj_type_id_str', 'ext_obj_encoding' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_extension_object_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5385,24 +5511,27 @@ filter { # opcua_binary_variant_metadata.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_variant_metadata" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][variant_source_data_link_id]} %{[zeek_cols][variant_data_source]} %{[zeek_cols][variant_data_source_str]} %{[zeek_cols][dara_variant_encoding_mask]} %{[zeek_cols][data_variant_data_type]} %{[zeek_cols][data_variant_data_type_str]} %{[zeek_cols][built_in_data_type]} %{[zeek_cols][built_in_data_type_str]} %{[zeek_cols][variant_data_link_id]} %{[zeek_cols][variant_data_array_dim]} %{[zeek_cols][variant_data_array_multi_dim_link_id]}" - } - } + if "_jsonparsesuccess" in [tags] { - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_variant_metadata" + } else { + dissect { + id => "dissect_zeek_opcua_binary_variant_metadata" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][variant_source_data_link_id]} %{[zeek_cols][variant_data_source]} %{[zeek_cols][variant_data_source_str]} %{[zeek_cols][dara_variant_encoding_mask]} %{[zeek_cols][data_variant_data_type]} %{[zeek_cols][data_variant_data_type_str]} %{[zeek_cols][built_in_data_type]} %{[zeek_cols][built_in_data_type_str]} %{[zeek_cols][variant_data_link_id]} %{[zeek_cols][variant_data_array_dim]} %{[zeek_cols][variant_data_array_multi_dim_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_variant_metadata" - init => "@zeek_opcua_binary_variant_metadata_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'variant_source_data_link_id', 'variant_data_source', 'variant_data_source_str', 'dara_variant_encoding_mask', 'data_variant_data_type', 'data_variant_data_type_str', 'built_in_data_type', 'built_in_data_type_str', 'variant_data_link_id', 'variant_data_array_dim', 'variant_data_array_multi_dim_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_metadata_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_variant_metadata" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_variant_metadata" + init => "@zeek_opcua_binary_variant_metadata_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'variant_source_data_link_id', 'variant_data_source', 'variant_data_source_str', 'dara_variant_encoding_mask', 'data_variant_data_type', 'data_variant_data_type_str', 'built_in_data_type', 'built_in_data_type_str', 'variant_data_link_id', 'variant_data_array_dim', 'variant_data_array_multi_dim_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_metadata_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5417,16 +5546,21 @@ filter { } else { # some other unknown zeek opcua- log file. should start with ts at least! - csv { - id => "csv_zeek_unknown_opcua" - columns => ["ts"] - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - separator => " " - # there's no way to *disable* the csv quote char, so set it to something we'll never see - quote_char => " " + if "_jsonparsesuccess" in [tags] { - target => "[zeek_cols]" + } else { + csv { + id => "csv_zeek_unknown_opcua" + columns => ["ts"] + + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + separator => " " + # there's no way to *disable* the csv quote char, so set it to something we'll never see + quote_char => " " + + target => "[zeek_cols]" + } } mutate { id => "mutate_add_tag_zeek_unknown_opcua" @@ -5445,23 +5579,27 @@ filter { # Zeek Logging analyzer confirmations and violations into analyzer.log # https://docs.zeek.org/en/master/scripts/base/frameworks/analyzer/logging.zeek.html - dissect { - id => "dissect_zeek_diagnostic_analyzer" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][cause]} %{[zeek_cols][analyzer_kind]} %{[zeek_cols][analyzer_name]} %{[zeek_cols][uid]} %{[zeek_cols][fuid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][failure_reason]} %{[zeek_cols][failure_data]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_diagnostic_analyzer" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_diagnostic_analyzer" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][cause]} %{[zeek_cols][analyzer_kind]} %{[zeek_cols][analyzer_name]} %{[zeek_cols][uid]} %{[zeek_cols][fuid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][failure_reason]} %{[zeek_cols][failure_data]}" + } } - ruby { - id => "ruby_zip_zeek_diagnostic_analyzer" - init => "@zeek_diagnostic_analyzer_field_names = [ 'ts', 'cause', 'analyzer_kind', 'analyzer_name', 'uid', 'fuid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'failure_reason', 'failure_data' ]" - code => "event.set('[zeek_cols]', @zeek_diagnostic_analyzer_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_analyzer" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_analyzer" + init => "@zeek_diagnostic_analyzer_field_names = [ 'ts', 'cause', 'analyzer_kind', 'analyzer_name', 'uid', 'fuid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'failure_reason', 'failure_data' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_analyzer_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5473,23 +5611,27 @@ filter { # broker.log # https://docs.zeek.org/en/master/scripts/base/frameworks/broker/log.zeek.html - dissect { - id => "dissect_zeek_diagnostic_broker" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][event_type]} %{[zeek_cols][event_action]} %{[zeek_cols][peer_ip]} %{[zeek_cols][peer_port]} %{[zeek_cols][peer_message]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_diagnostic_broker" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_diagnostic_broker" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][event_type]} %{[zeek_cols][event_action]} %{[zeek_cols][peer_ip]} %{[zeek_cols][peer_port]} %{[zeek_cols][peer_message]}" + } } - ruby { - id => "ruby_zip_zeek_diagnostic_broker" - init => "@zeek_diagnostic_broker_field_names = [ 'ts', 'event_type', 'event_action', 'peer_ip', 'peer_port', 'peer_message' ]" - code => "event.set('[zeek_cols]', @zeek_diagnostic_broker_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_broker" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_broker" + init => "@zeek_diagnostic_broker_field_names = [ 'ts', 'event_type', 'event_action', 'peer_ip', 'peer_port', 'peer_message' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_broker_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5502,23 +5644,27 @@ filter { # Reports analysis of missing traffic. Zeek bases its conclusions on analysis of TCP sequence numbers. # https://docs.zeek.org/en/master/logs/capture-loss-and-reporter.html - dissect { - id => "dissect_zeek_diagnostic_capture_loss" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][ts_delta]} %{[zeek_cols][peer]} %{[zeek_cols][gaps]} %{[zeek_cols][acks]} %{[zeek_cols][percent_lost]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_diagnostic_capture_loss" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_diagnostic_capture_loss" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][ts_delta]} %{[zeek_cols][peer]} %{[zeek_cols][gaps]} %{[zeek_cols][acks]} %{[zeek_cols][percent_lost]}" + } } - ruby { - id => "ruby_zip_zeek_diagnostic_capture_loss" - init => "@zeek_diagnostic_capture_loss_field_names = [ 'ts', 'ts_delta', 'peer', 'gaps', 'acks', 'percent_lost' ]" - code => "event.set('[zeek_cols]', @zeek_diagnostic_capture_loss_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_capture_loss" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_capture_loss" + init => "@zeek_diagnostic_capture_loss_field_names = [ 'ts', 'ts_delta', 'peer', 'gaps', 'acks', 'percent_lost' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_capture_loss_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5531,23 +5677,27 @@ filter { # Logging for establishing and controlling a cluster of Zeek instances # https://docs.zeek.org/en/master/scripts/base/frameworks/cluster/main.zeek.html#type-Cluster::Info - dissect { - id => "dissect_zeek_diagnostic_cluster" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][node]} %{[zeek_cols][node_message]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_diagnostic_cluster" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_diagnostic_cluster" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][node]} %{[zeek_cols][node_message]}" + } } - ruby { - id => "ruby_zip_zeek_diagnostic_cluster" - init => "@zeek_diagnostic_cluster_field_names = [ 'ts', 'node', 'node_message' ]" - code => "event.set('[zeek_cols]', @zeek_diagnostic_cluster_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_cluster" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_cluster" + init => "@zeek_diagnostic_cluster_field_names = [ 'ts', 'node', 'node_message' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_cluster_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5560,23 +5710,27 @@ filter { # Logging for Zeek configuration changes # https://docs.zeek.org/en/master/scripts/base/frameworks/config/main.zeek.html#type-Config::Info - dissect { - id => "dissect_zeek_diagnostic_config" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][value_name]} %{[zeek_cols][value_old]} %{[zeek_cols][value_new]} %{[zeek_cols][location]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_diagnostic_config" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_diagnostic_config" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][value_name]} %{[zeek_cols][value_old]} %{[zeek_cols][value_new]} %{[zeek_cols][location]}" + } } - ruby { - id => "ruby_zip_zeek_diagnostic_config" - init => "@zeek_diagnostic_config_field_names = [ 'ts', 'value_name', 'value_old', 'value_new', 'location' ]" - code => "event.set('[zeek_cols]', @zeek_diagnostic_config_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_config" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_config" + init => "@zeek_diagnostic_config_field_names = [ 'ts', 'value_name', 'value_old', 'value_new', 'location' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_config_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5588,23 +5742,27 @@ filter { # packet_filter.log # https://docs.zeek.org/en/master/scripts/base/frameworks/packet-filter/main.zeek.html#type-PacketFilter::Info - dissect { - id => "dissect_zeek_diagnostic_packet_filter" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][node]} %{[zeek_cols][filter]} %{[zeek_cols][init]} %{[zeek_cols][success]} %{[zeek_cols][failure_reason]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_diagnostic_packet_filter" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_diagnostic_packet_filter" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][node]} %{[zeek_cols][filter]} %{[zeek_cols][init]} %{[zeek_cols][success]} %{[zeek_cols][failure_reason]}" + } } - ruby { - id => "ruby_zip_zeek_diagnostic_packet_filter" - init => "@zeek_diagnostic_packet_filter_field_names = [ 'ts', 'node', 'filter', 'init', 'success', 'failure_reason' ]" - code => "event.set('[zeek_cols]', @zeek_diagnostic_packet_filter_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_packet_filter" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_packet_filter" + init => "@zeek_diagnostic_packet_filter_field_names = [ 'ts', 'node', 'filter', 'init', 'success', 'failure_reason' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_packet_filter_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5616,23 +5774,27 @@ filter { # print.log # https://docs.zeek.org/en/master/scripts/base/frameworks/logging/main.zeek.html#type-Log::PrintLogInfo - dissect { - id => "dissect_zeek_diagnostic_print" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][vals]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_diagnostic_print" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_diagnostic_print" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][vals]}" + } } - ruby { - id => "ruby_zip_zeek_diagnostic_print" - init => "@zeek_diagnostic_print_field_names = [ 'ts', 'vals' ]" - code => "event.set('[zeek_cols]', @zeek_diagnostic_print_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_print" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_print" + init => "@zeek_diagnostic_print_field_names = [ 'ts', 'vals' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_print_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5648,23 +5810,27 @@ filter { # reporter.log # https://docs.zeek.org/en/master/scripts/base/frameworks/reporter/main.zeek.html#type-Reporter::Info - dissect { - id => "dissect_zeek_diagnostic_reporter" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][level]} %{[zeek_cols][msg]} %{[zeek_cols][location]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_diagnostic_reporter" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_diagnostic_reporter" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][level]} %{[zeek_cols][msg]} %{[zeek_cols][location]}" + } } - ruby { - id => "ruby_zip_zeek_diagnostic_reporter" - init => "@zeek_diagnostic_reporter_field_names = [ 'ts', 'node', 'filter', 'init', 'success', 'failure_reason' ]" - code => "event.set('[zeek_cols]', @zeek_diagnostic_reporter_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_reporter" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_reporter" + init => "@zeek_diagnostic_reporter_field_names = [ 'ts', 'node', 'filter', 'init', 'success', 'failure_reason' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_reporter_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5676,23 +5842,27 @@ filter { # stats.log # https://docs.zeek.org/en/master/scripts/policy/misc/stats.zeek.html#type-Stats::Info - dissect { - id => "dissect_zeek_diagnostic_stats" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][peer]} %{[zeek_cols][mem]} %{[zeek_cols][pkts_proc]} %{[zeek_cols][bytes_recv]} %{[zeek_cols][pkts_dropped]} %{[zeek_cols][pkts_link]} %{[zeek_cols][pkt_lag]} %{[zeek_cols][pkts_filtered]} %{[zeek_cols][events_proc]} %{[zeek_cols][events_queued]} %{[zeek_cols][active_tcp_conns]} %{[zeek_cols][active_udp_conns]} %{[zeek_cols][active_icmp_conns]} %{[zeek_cols][tcp_conns]} %{[zeek_cols][udp_conns]} %{[zeek_cols][icmp_conns]} %{[zeek_cols][timers]} %{[zeek_cols][active_timers]} %{[zeek_cols][files]} %{[zeek_cols][active_files]} %{[zeek_cols][dns_requests]} %{[zeek_cols][active_dns_requests]} %{[zeek_cols][reassem_tcp_size]} %{[zeek_cols][reassem_file_size]} %{[zeek_cols][reassem_frag_size]} %{[zeek_cols][reassem_unknown_size]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_diagnostic_stats" + if "_jsonparsesuccess" in [tags] { + + } else { + dissect { + id => "dissect_zeek_diagnostic_stats" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][peer]} %{[zeek_cols][mem]} %{[zeek_cols][pkts_proc]} %{[zeek_cols][bytes_recv]} %{[zeek_cols][pkts_dropped]} %{[zeek_cols][pkts_link]} %{[zeek_cols][pkt_lag]} %{[zeek_cols][pkts_filtered]} %{[zeek_cols][events_proc]} %{[zeek_cols][events_queued]} %{[zeek_cols][active_tcp_conns]} %{[zeek_cols][active_udp_conns]} %{[zeek_cols][active_icmp_conns]} %{[zeek_cols][tcp_conns]} %{[zeek_cols][udp_conns]} %{[zeek_cols][icmp_conns]} %{[zeek_cols][timers]} %{[zeek_cols][active_timers]} %{[zeek_cols][files]} %{[zeek_cols][active_files]} %{[zeek_cols][dns_requests]} %{[zeek_cols][active_dns_requests]} %{[zeek_cols][reassem_tcp_size]} %{[zeek_cols][reassem_file_size]} %{[zeek_cols][reassem_frag_size]} %{[zeek_cols][reassem_unknown_size]}" + } } - ruby { - id => "ruby_zip_zeek_diagnostic_stats" - init => "@zeek_diagnostic_stats_field_names = [ 'ts', 'peer', 'mem', 'pkts_proc', 'bytes_recv', 'pkts_dropped', 'pkts_link', 'pkt_lag', 'pkts_filtered', 'events_proc', 'events_queued', 'active_tcp_conns', 'active_udp_conns', 'active_icmp_conns', 'tcp_conns', 'udp_conns', 'icmp_conns', 'timers', 'active_timers', 'files', 'active_files', 'dns_requests', 'active_dns_requests', 'reassem_tcp_size', 'reassem_file_size', 'reassem_frag_size', 'reassem_unknown_size' ]" - code => "event.set('[zeek_cols]', @zeek_diagnostic_stats_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_stats" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_stats" + init => "@zeek_diagnostic_stats_field_names = [ 'ts', 'peer', 'mem', 'pkts_proc', 'bytes_recv', 'pkts_dropped', 'pkts_link', 'pkt_lag', 'pkts_filtered', 'events_proc', 'events_queued', 'active_tcp_conns', 'active_udp_conns', 'active_icmp_conns', 'tcp_conns', 'udp_conns', 'icmp_conns', 'timers', 'active_timers', 'files', 'active_files', 'dns_requests', 'active_dns_requests', 'reassem_tcp_size', 'reassem_file_size', 'reassem_frag_size', 'reassem_unknown_size' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_stats_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5700,17 +5870,22 @@ filter { add_tag => [ "_zeekdiagnostic" ] } } else { - # some other unknown zeek log file. should start with ts at least! - csv { - id => "csv_zeek_unknown" - columns => ["ts"] - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - separator => " " - # there's no way to *disable* the csv quote char, so set it to something we'll never see - quote_char => " " + if "_jsonparsesuccess" in [tags] { + + } else { + # some other unknown zeek log file. should start with ts at least! + csv { + id => "csv_zeek_unknown" + columns => ["ts"] + + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + separator => " " + # there's no way to *disable* the csv quote char, so set it to something we'll never see + quote_char => " " - target => "[zeek_cols]" + target => "[zeek_cols]" + } } mutate { id => "mutate_add_tag_zeek_unknown" From cea233acb678384f83b8ee09dc4be5abe997b6a9 Mon Sep 17 00:00:00 2001 From: SG Date: Fri, 16 Feb 2024 14:48:08 -0700 Subject: [PATCH 06/79] for supporting JSON logs from Zeek (idaholab/Malcolm#65); almost certainly broken at this point --- config/zeek.env.example | 2 ++ .../config/includes.chroot/usr/local/etc/zeek/local.zeek | 5 +++++ hedgehog-iso/interface/sensor_ctl/control_vars.conf | 1 + zeek/config/local.zeek | 5 +++++ 4 files changed, 13 insertions(+) diff --git a/config/zeek.env.example b/config/zeek.env.example index 06c72e8de..880e583e3 100644 --- a/config/zeek.env.example +++ b/config/zeek.env.example @@ -3,6 +3,8 @@ # Note that by default, Zeek considers IANA-registered private address space # such as 10/8 and 192.168/16 site-local. ZEEK_LOCAL_NETS= +# Set to true to indicate that Zeek should output logs in JSON format +ZEEK_JSON= # Specifies the value for Zeek's Intel::item_expiration timeout (-1min to disable) ZEEK_INTEL_ITEM_EXPIRATION=-1min # When querying a TAXII or MISP feed, only process threat indicators that have diff --git a/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek index 717c4cd82..0d27093e6 100644 --- a/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek +++ b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek @@ -46,6 +46,11 @@ global disable_ics_synchrophasor = (getenv("ZEEK_DISABLE_ICS_SYNCHROPHASOR") == redef Broker::default_listen_address = "127.0.0.1"; redef ignore_checksums = T; +global json_format = (getenv("ZEEK_JSON") == true_regex) ? T : F; +@if (json_format) + redef LogAscii::use_json = T; +@endif + @load tuning/defaults @load frameworks/software/vulnerable @load frameworks/software/version-changes diff --git a/hedgehog-iso/interface/sensor_ctl/control_vars.conf b/hedgehog-iso/interface/sensor_ctl/control_vars.conf index 956d6daa5..16998d5c4 100644 --- a/hedgehog-iso/interface/sensor_ctl/control_vars.conf +++ b/hedgehog-iso/interface/sensor_ctl/control_vars.conf @@ -47,6 +47,7 @@ export ZEEK_LB_METHOD=custom export ZEEK_AF_PACKET_BUFFER_SIZE=67108864 export ZEEK_LOCAL_NETS= +export ZEEK_JSON= export ZEEK_RULESET=local export ZEEK_INTEL_ITEM_EXPIRATION=-1min export ZEEK_INTEL_FEED_SINCE= diff --git a/zeek/config/local.zeek b/zeek/config/local.zeek index d1f89808e..8f04cf0e8 100644 --- a/zeek/config/local.zeek +++ b/zeek/config/local.zeek @@ -46,6 +46,11 @@ global disable_ics_synchrophasor = (getenv("ZEEK_DISABLE_ICS_SYNCHROPHASOR") == redef Broker::default_listen_address = "127.0.0.1"; redef ignore_checksums = T; +global json_format = (getenv("ZEEK_JSON") == true_regex) ? T : F; +@if (json_format) + redef LogAscii::use_json = T; +@endif + @load tuning/defaults @load frameworks/software/vulnerable @load frameworks/software/version-changes From cc90ca9d55d5089b9182d1fc5b3df11326e5bee3 Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 20 Feb 2024 12:03:59 -0700 Subject: [PATCH 07/79] for supporting JSON logs from Zeek (idaholab/Malcolm#65); almost certainly broken at this point --- logstash/pipelines/zeek/11_zeek_parse.conf | 806 ++++++++------------ logstash/pipelines/zeek/12_zeek_mutate.conf | 350 +++------ 2 files changed, 402 insertions(+), 754 deletions(-) diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index ed6dfda3a..8ea1bbff5 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -24,6 +24,36 @@ filter { add_tag => [ "_jsonparsesuccess" ] } } + + # in JSON, do some global renaming of common fields to make them match the names we'd + # be assigning to them if we were reading TSV + if ("_jsonparsesuccess" in [tags]) { + if ("" in [zeek_cols][is_orig]) and (([zeek_cols][source_h]) or ([zeek_cols][destination_h])) { + # some of the ICSNPP parsers do an interesting thing to handle source and destination fields + # (see https://github.com/cisagov/icsnpp-bacnet/?tab=readme-ov-file#source-and-destination-fields) + # so check for and handle those first + mutate { + id => "mutate_rename_zeek_json_common_reversed_direction_fields" + rename => { "[zeek_cols][id.orig_h]" => "[zeek_cols][drop_orig_h]" } + rename => { "[zeek_cols][id.orig_p]" => "[zeek_cols][drop_orig_p]" } + rename => { "[zeek_cols][id.resp_h]" => "[zeek_cols][drop_resp_h]" } + rename => { "[zeek_cols][id.resp_p]" => "[zeek_cols][drop_resp_p]" } + rename => { "[zeek_cols][source_h]" => "[zeek_cols][orig_h]" } + rename => { "[zeek_cols][source_p]" => "[zeek_cols][orig_p]" } + rename => { "[zeek_cols][destination_h]" => "[zeek_cols][resp_h]" } + rename => { "[zeek_cols][destination_p]" => "[zeek_cols][resp_p]" } + } + } else { + mutate { + id => "mutate_rename_zeek_json_common_fields" + rename => { "[zeek_cols][id.orig_h]" => "[zeek_cols][orig_h]" } + rename => { "[zeek_cols][id.orig_p]" => "[zeek_cols][orig_p]" } + rename => { "[zeek_cols][id.resp_h]" => "[zeek_cols][resp_h]" } + rename => { "[zeek_cols][id.resp_p]" => "[zeek_cols][resp_p]" } + } + } + } + # The Dissect is WAY faster than CSV, and quite a bit faster than mutate.split. However, it # is not as flexible when it comes to missing or extra columns # (See https://github.com/logstash-plugins/logstash-filter-dissect/issues/62) @@ -40,9 +70,7 @@ filter { # conn.log # https://docs.zeek.org/en/stable/scripts/base/protocols/conn/main.zeek.html#type-Conn::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_conn_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -104,8 +132,8 @@ filter { add_field => { "[server][bytes]" => "%{[zeek_cols][resp_bytes]}" } } } if ([zeek_cols][tunnel_parents]) and ([zeek_cols][tunnel_parents] != '(empty)') and ([zeek_cols][tunnel_parents] != '-') and ([zeek_cols][tunnel_parents] != '') { - mutate { id => "mutate_split_zeek_tunnel_parents" - split => { "[zeek_cols][tunnel_parents]" => "," } } + if ("_jsonparsesuccess" not in [tags]) { mutate { id => "mutate_split_zeek_tunnel_parents" + split => { "[zeek_cols][tunnel_parents]" => "," } } } mutate { id => "mutate_add_field_zeek_conn_rootId" add_field => { "[rootId]" => "%{[zeek_cols][tunnel_parents][0]}" } } } @@ -115,9 +143,7 @@ filter { # bacnet.log # https://github.com/cisagov/ICSNPP - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_bacnet" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -152,9 +178,7 @@ filter { ############################################################################################################################# # bestguess.log - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_bestguess" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -184,9 +208,7 @@ filter { # bsap_ip_header.log # https://github.com/cisagov/ICSNPP - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_bsap_ip_header" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -222,9 +244,7 @@ filter { # bsap_ip_rdb.log # https://github.com/cisagov/ICSNPP - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_bsap_ip_rdb" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -260,9 +280,7 @@ filter { # bsap_serial_header.log # https://github.com/cisagov/ICSNPP - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_bsap_serial_header" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -298,9 +316,7 @@ filter { # bsap_serial_rdb.log # https://github.com/cisagov/ICSNPP - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_bsap_serial_rdb" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -336,9 +352,7 @@ filter { # bsap_serial_rdb_ext.log # https://github.com/cisagov/ICSNPP - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_bsap_serial_rdb_ext" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -380,9 +394,7 @@ filter { # bacnet_device_control.log # https://github.com/cisagov/ICSNPP - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_bacnet_device_control" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -418,9 +430,7 @@ filter { # bacnet_discovery.log # https://github.com/cisagov/ICSNPP - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_bacnet_discovery" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -456,9 +466,7 @@ filter { # bacnet_property.log # https://github.com/cisagov/ICSNPP - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_bacnet_property" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -496,9 +504,7 @@ filter { # # todo: class_id, instance_id is a hex integer, should it be converted to an integer? - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_cip" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -533,9 +539,7 @@ filter { # # TODO: device_status and device_state are a hex int string, convert to int? - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_cip_identity" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -551,7 +555,7 @@ filter { } ruby { id => "ruby_zip_zeek_cip_identity" - init => "@zeek_cip_identity_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'encapsulation_version', 'socket_address', 'socket_port', 'vendor_id', 'vendor_name', 'device_type_id', 'device_type_name', 'product_code', 'device_status', 'serial_number', 'product_name', 'device_state' ]" + init => "@zeek_cip_identity_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'encapsulation_version', 'socket_address', 'socket_port', 'vendor_id', 'vendor_name', 'device_type_id', 'device_type_name', 'product_code', 'revision', 'device_status', 'serial_number', 'product_name', 'device_state' ]" code => "event.set('[zeek_cols]', @zeek_cip_identity_field_names.zip(event.get('[message]')).to_h)" } } @@ -569,9 +573,7 @@ filter { # https://github.com/cisagov/ICSNPP # - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_cip_io" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -604,9 +606,7 @@ filter { # dce_rpc.log # https://docs.zeek.org/en/stable/scripts/base/protocols/dce-rpc/main.zeek.html#type-DCE_RPC::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_dce_rpc" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -641,7 +641,15 @@ filter { # dhcp.log # https://docs.zeek.org/en/stable/scripts/base/protocols/dhcp/main.zeek.html#type-DHCP::Info - if "_jsonparsesuccess" in [tags] { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_dhcp_fields" + rename => { "[zeek_cols][client_addr]" => "[zeek_cols][orig_h]" } + rename => { "[zeek_cols][server_addr]" => "[zeek_cols][orig_p]" } + rename => { "[zeek_cols][mac]" => "[zeek_cols][orig_l2_addr]" } + rename => { "[zeek_cols][requested_addr]" => "[zeek_cols][requested_ip]" } + rename => { "[zeek_cols][assigned_addr]" => "[zeek_cols][assigned_ip]" } + } } else { dissect { @@ -663,12 +671,16 @@ filter { code => "event.set('[zeek_cols]', @zeek_dhcp_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_dhcp_msg_types" + split => { "[zeek_cols][msg_types]" => "," } } + if ([zeek_cols][uid]) and + ([zeek_cols][uid] != '(empty)') and + ([zeek_cols][uid] != '') { + mutate { id => "mutate_split_zeek_dhcp_uids" + split => { "[zeek_cols][uid]" => "," } } + } } - if ([zeek_cols][uid]) and ([zeek_cols][uid] != '(empty)') and ([zeek_cols][uid] != '') { - mutate { id => "mutate_split_zeek_dhcp_uids" - split => { "[zeek_cols][uid]" => "," } } - } if ([zeek_cols][uid] and [zeek_cols][uid][0]) { mutate { id => "mutate_add_field_zeek_dhcp_uids" add_field => { "[rootId]" => "%{[zeek_cols][uid][0]}" } } @@ -697,9 +709,7 @@ filter { # dnp3.log # https://docs.zeek.org/en/stable/scripts/base/protocols/dnp3/main.zeek.html#type-DNP3::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_dnp3" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -732,9 +742,7 @@ filter { # dnp3_control.log # https://github.com/cisagov/ICSNPP - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_dnp3_control" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -767,9 +775,7 @@ filter { # dnp3_objects.log # https://github.com/cisagov/ICSNPP - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_dnp3_objects" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -802,9 +808,7 @@ filter { # dns.log # https://docs.zeek.org/en/stable/scripts/base/protocols/dns/main.zeek.html#type-DNS::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_dns" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -824,6 +828,9 @@ filter { code => "event.set('[zeek_cols]', @zeek_dns_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_dns_commas" + split => { "[zeek_cols][TTLs]" => "," + "[zeek_cols][answers]" => "," } } } # remove C_ prefix from qclass_name @@ -840,9 +847,7 @@ filter { # dpd.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/dpd/main.zeek.html#type-DPD::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_dpd" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -885,9 +890,7 @@ filter { # https://github.com/cisagov/ICSNPP # - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_enip" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -920,9 +923,7 @@ filter { # ecat_registers.log # https://github.com/cisagov/icsnpp-ethercat - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_ecat_registers" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -957,9 +958,7 @@ filter { # ecat_log_address.log # https://github.com/cisagov/icsnpp-ethercat - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_ecat_log_address" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -994,9 +993,7 @@ filter { # ecat_dev_info.log # https://github.com/cisagov/icsnpp-ethercat - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_ecat_dev_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1031,9 +1028,7 @@ filter { # ecat_aoe_info.log # https://github.com/cisagov/icsnpp-ethercat - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_ecat_aoe_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1068,9 +1063,7 @@ filter { # ecat_coe_info.log # https://github.com/cisagov/icsnpp-ethercat - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_ecat_coe_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1105,9 +1098,7 @@ filter { # ecat_foe_info.log # https://github.com/cisagov/icsnpp-ethercat - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_ecat_foe_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1142,9 +1133,7 @@ filter { # ecat_soe_info.log # https://github.com/cisagov/icsnpp-ethercat - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_ecat_soe_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1183,9 +1172,7 @@ filter { # to be handled bdrop_zeek_ignored_source above, as it is including ALL arps and not just those # from ethercat traffic which can be misleading (i.e., indicating ecat traffic where there is none) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_ecat_arp_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1221,9 +1208,7 @@ filter { # files.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/files/main.zeek.html#type-Files::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_v51_files_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1243,19 +1228,21 @@ filter { code => "event.set('[zeek_cols]', @zeek_files_field_names.zip(event.get('[message]')).to_h)" } } - } - - if ([zeek_cols][conn_uids]) and ([zeek_cols][conn_uids] != '(empty)') and ([zeek_cols][conn_uids] != '') { - mutate { id => "mutate_split_zeek_files_conn_uids" - split => { "[zeek_cols][conn_uids]" => "," } } - } - if ([zeek_cols][tx_hosts]) and ([zeek_cols][tx_hosts] != '(empty)') and ([zeek_cols][tx_hosts] != '') { - mutate { id => "mutate_split_zeek_files_tx_hosts" - split => { "[zeek_cols][tx_hosts]" => "," } } - } - if ([zeek_cols][rx_hosts]) and ([zeek_cols][rx_hosts] != '(empty)') and ([zeek_cols][rx_hosts] != '') { - mutate { id => "mutate_split_zeek_files_rx_hosts" - split => { "[zeek_cols][rx_hosts]" => "," } } + if ([zeek_cols][conn_uids]) and ([zeek_cols][conn_uids] != '(empty)') and ([zeek_cols][conn_uids] != '') { + mutate { id => "mutate_split_zeek_files_conn_uids" + split => { "[zeek_cols][conn_uids]" => "," } } + } + if ([zeek_cols][tx_hosts]) and ([zeek_cols][tx_hosts] != '(empty)') and ([zeek_cols][tx_hosts] != '') { + mutate { id => "mutate_split_zeek_files_tx_hosts" + split => { "[zeek_cols][tx_hosts]" => "," } } + } + if ([zeek_cols][rx_hosts]) and ([zeek_cols][rx_hosts] != '(empty)') and ([zeek_cols][rx_hosts] != '') { + mutate { id => "mutate_split_zeek_files_rx_hosts" + split => { "[zeek_cols][rx_hosts]" => "," } } + } + mutate { id => "mutate_split_zeek_files_parent_fuid_and_analyzers" + split => { "[zeek_cols][parent_fuid]" => "," + "[zeek_cols][analyzers]" => "," } } } if ([zeek_cols][conn_uids] and [zeek_cols][conn_uids][0]) { @@ -1287,9 +1274,7 @@ filter { # ftp.log # https://docs.zeek.org/en/stable/scripts/base/protocols/ftp/info.zeek.html#type-FTP::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_ftp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1324,9 +1309,7 @@ filter { # genisys.log # https://github.com/cisagov/icsnpp-genisys - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_genisys" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1361,9 +1344,7 @@ filter { # gquic.log # https://github.com/salesforce/GQUIC_Protocol_Analyzer/blob/master/scripts/Salesforce/GQUIC/main.bro - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_gquic" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1398,9 +1379,7 @@ filter { # http.log # https://docs.zeek.org/en/stable/scripts/base/protocols/http/main.zeek.html#type-HTTP::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_http_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1420,6 +1399,15 @@ filter { code => "event.set('[zeek_cols]', @zeek_http_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_http_commas" + split => { "[zeek_cols][orig_fuids]" => "," + "[zeek_cols][orig_filenames]" => "," + "[zeek_cols][orig_mime_types]" => "," + "[zeek_cols][resp_fuids]" => "," + "[zeek_cols][resp_filenames]" => "," + "[zeek_cols][resp_mime_types]" => "," + "[zeek_cols][proxied]" => "," + "[zeek_cols][tags]" => "," } } } mutate { @@ -1435,9 +1423,7 @@ filter { # intel.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_intel" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1457,6 +1443,10 @@ filter { code => "event.set('[zeek_cols]', @zeek_intel_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_intel_commas" + split => { "[zeek_cols][sources]" => "," + "[zeek_cols][matched]" => "," + "[zeek_cols][cif_tags]" => "," } } } } else if ([log_source] == "ipsec") { @@ -1464,9 +1454,7 @@ filter { # ipsec.log # https://github.com/corelight/zeek-spicy-ipsec/blob/master/analyzer/main.zeek - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_ipsec" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1486,6 +1474,14 @@ filter { code => "event.set('[zeek_cols]', @zeek_ipsec_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_ipsec_commas" + split => { "[zeek_cols][vendor_ids]" => "," + "[zeek_cols][notify_messages]" => "," + "[zeek_cols][transforms]" => "," + "[zeek_cols][ke_dh_groups]" => "," + "[zeek_cols][proposals]" => "," + "[zeek_cols][certificates]" => "," + "[zeek_cols][transform_attributes]" => "," } } } mutate { @@ -1500,9 +1496,7 @@ filter { # irc.log # https://docs.zeek.org/en/stable/scripts/base/protocols/irc/main.zeek.html#type-IRC::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_irc" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1537,9 +1531,7 @@ filter { # cotp.log # https://github.com/cisagov/icsnpp-s7comm - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_cotp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1575,9 +1567,7 @@ filter { # kerberos.log # https://docs.zeek.org/en/stable/scripts/base/protocols/krb/main.zeek.html#type-KRB::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_kerberos" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1597,6 +1587,9 @@ filter { code => "event.set('[zeek_cols]', @zeek_kerberos_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_kerberos_commas" + split => { "[zeek_cols][client_cert_fuid]" => "," + "[zeek_cols][server_cert_fuid]" => "," } } } mutate { @@ -1609,9 +1602,7 @@ filter { # known_certs.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/ssl/known-certs.zeek.html#type-Known::CertsInfo - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_known_certs" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1646,9 +1637,7 @@ filter { # known_hosts.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/conn/known-hosts.zeek.html#type-Known::HostsInfo - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_known_hosts" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1675,9 +1664,7 @@ filter { # known_modbus.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/modbus/known-masters-slaves.zeek.html#type-Known::ModbusInfo - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_known_modbus" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1716,9 +1703,7 @@ filter { # known_services.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/conn/known-services.zeek.html#type-Known::ServicesInfo - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_known_services" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1766,9 +1751,7 @@ filter { # ldap.log # main.zeek (https://docs.zeek.org/en/master/scripts/base/protocols/ldap/main.zeek.html) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_ldap" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1788,6 +1771,12 @@ filter { code => "event.set('[zeek_cols]', @zeek_ldap_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_ldap_commas" + split => { "[zeek_cols][operation]" => "," + "[zeek_cols][result_code]" => "," + "[zeek_cols][result_message]" => "," + "[zeek_cols][object]" => "," + "[zeek_cols][argument]" => "," } } } mutate { @@ -1803,9 +1792,7 @@ filter { # ldap_search.log # main.zeek (https://docs.zeek.org/en/master/scripts/base/protocols/ldap/main.zeek.html) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_ldap_search" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1825,6 +1812,13 @@ filter { code => "event.set('[zeek_cols]', @zeek_ldap_search_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_ldap_search_commas" + split => { "[zeek_cols][scope]" => "," + "[zeek_cols][deref]" => "," + "[zeek_cols][attributes]" => "," + "[zeek_cols][base_object]" => "," + "[zeek_cols][result_code]" => "," + "[zeek_cols][result_message]" => "," } } } mutate { @@ -1840,9 +1834,7 @@ filter { # login.log # custom login.log module (rudimentary, telnet/rlogin/rsh analyzers are old and not the greatest) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_login" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1869,9 +1861,7 @@ filter { # modbus.log # https://docs.zeek.org/en/stable/scripts/base/protocols/modbus/main.zeek.html#type-Modbus::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_modbus" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1904,9 +1894,7 @@ filter { # modbus_detailed.log # main.zeek (https://github.com/cisagov/icsnpp-modbus) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_modbus_detailed" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1926,6 +1914,8 @@ filter { code => "event.set('[zeek_cols]', @zeek_modbus_detailed_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_modbus_detailed_values" + split => { "[zeek_cols][values]" => "," } } } mutate { @@ -1941,9 +1931,7 @@ filter { # modbus_mask_write_register.log # main.zeek (https://github.com/cisagov/icsnpp-modbus) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_modbus_mask_write_register" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1978,9 +1966,7 @@ filter { # modbus_read_device_identification.log # main.zeek (https://github.com/cisagov/icsnpp-modbus) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_modbus_read_device_identification" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2015,9 +2001,7 @@ filter { # modbus_read_write_multiple_registers.log # main.zeek (https://github.com/cisagov/icsnpp-modbus) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_modbus_read_write_multiple_registers" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2037,6 +2021,9 @@ filter { code => "event.set('[zeek_cols]', @zeek_modbus_read_write_multiple_registers_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_modbus_read_write_multiple_registers_read_commas" + split => { "[zeek_cols][read_registers]" => "," + "[zeek_cols][write_registers]" => "," } } } mutate { @@ -2052,9 +2039,7 @@ filter { # mqtt_connect.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/mqtt/main.zeek.html#type-MQTT::ConnectInfo - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_mqtt_connect" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2089,9 +2074,7 @@ filter { # mqtt_publish.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/mqtt/main.zeek.html#type-MQTT::PublishInfo - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_mqtt_publish" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2126,9 +2109,7 @@ filter { # mqtt_subscribe.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/mqtt/main.zeek.html#type-MQTT::SubscribeInfo - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_mqtt_subscribe" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2148,6 +2129,9 @@ filter { code => "event.set('[zeek_cols]', @zeek_mqtt_subscribe_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_mqtt_subscribe_commas" + split => { "[zeek_cols][topics]" => "," + "[zeek_cols][qos_levels]" => "," } } } mutate { @@ -2166,9 +2150,7 @@ filter { # mysql.log # https://docs.zeek.org/en/stable/scripts/base/protocols/mysql/main.zeek.html#type-MySQL::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_mysql" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2200,9 +2182,7 @@ filter { # notice.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/notice/main.zeek.html#type-Notice::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_notice_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2222,6 +2202,8 @@ filter { code => "event.set('[zeek_cols]', @zeek_notice_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_notice_actions" + split => { "[zeek_cols][actions]" => "," } } } if (![zeek_cols][orig_h]) or ([zeek_cols][orig_h] == '(empty)') or @@ -2245,9 +2227,7 @@ filter { # ntlm.log # https://docs.zeek.org/en/stable/scripts/base/protocols/ntlm/main.zeek.html#type-NTLM::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_ntlm_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2279,9 +2259,7 @@ filter { # ntp.log # https://docs.zeek.org/en/latest/scripts/base/protocols/ntp/main.zeek.html#type-NTP::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_ntp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2316,9 +2294,7 @@ filter { # ocsp.log # https://docs.zeek.org/en/stable/scripts/policy/files/x509/log-ocsp.zeek.html#type-OCSP::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_ocsp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2349,9 +2325,7 @@ filter { # ospf.log # https://github.com/corelight/zeek-spicy-ospf - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_ospf" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2372,6 +2346,14 @@ filter { code => "event.set('[zeek_cols]', @zeek_ospf_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_ospf_commas" + split => { "[zeek_cols][neighbors]" => "," + "[zeek_cols][routers]" => "," + "[zeek_cols][metrics]" => "," + "[zeek_cols][fwd_addrs]" => "," + "[zeek_cols][route_tags]" => "," + "[zeek_cols][link_prefixes]" => "," + "[zeek_cols][intra_prefixes]" => "," } } } mutate { @@ -2387,9 +2369,7 @@ filter { # pe.log # https://docs.zeek.org/en/stable/scripts/base/files/pe/main.zeek.html#type-PE::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_pe" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2409,6 +2389,8 @@ filter { code => "event.set('[zeek_cols]', @zeek_pe_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_pe_section_names" + split => { "[zeek_cols][section_names]" => "," } } } } else if ([log_source] == "profinet") { @@ -2416,9 +2398,7 @@ filter { # profinet.log # https://github.com/amzn/zeek-plugin-profinet/blob/master/scripts/main.zeek - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_profinet" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2451,9 +2431,7 @@ filter { # profinet_dce_rpc.log # https://github.com/amzn/zeek-plugin-profinet/blob/master/scripts/main.zeek - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_profinet_dce_rpc" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2486,9 +2464,7 @@ filter { # radius.log # https://docs.zeek.org/en/stable/scripts/base/protocols/radius/main.zeek.html#type-RADIUS::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_radius" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2520,9 +2496,7 @@ filter { # rdp.log # https://docs.zeek.org/en/stable/scripts/base/protocols/rdp/main.zeek.html#type-RDP::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_rdp_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2542,6 +2516,8 @@ filter { code => "event.set('[zeek_cols]', @zeek_rdp_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_rdp_client_channels" + split => { "[zeek_cols][client_channels]" => "," } } } mutate { @@ -2558,9 +2534,7 @@ filter { # rfb.log # https://docs.zeek.org/en/stable/scripts/base/protocols/rfb/main.zeek.html#type-RFB::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_rfb" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2592,9 +2566,7 @@ filter { # s7comm.log # https://github.com/cisagov/icsnpp-s7comm - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_s7comm" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2630,9 +2602,7 @@ filter { # s7comm_plus.log # https://github.com/cisagov/icsnpp-s7comm - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_s7comm_plus" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2668,9 +2638,7 @@ filter { # s7comm_read_szl.log # https://github.com/cisagov/icsnpp-s7comm - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_s7comm_read_szl" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2706,9 +2674,7 @@ filter { # s7comm_upload_download.log # https://github.com/cisagov/icsnpp-s7comm - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_s7comm_upload_download" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2743,9 +2709,7 @@ filter { ############################################################################################################################# # signatures.log - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_signatures" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2765,6 +2729,12 @@ filter { code => "event.set('[zeek_cols]', @zeek_signatures_field_names.zip(event.get('[message]')).to_h)" } } + if ("_carved" in [tags]) { + # Malcolm does some "special" stuff in zeek_carve_logger.py for file carving, sort of hijacking signatures.log for it: + # - _carved signature logs' sub_message contains fuid(s) comma-separated + mutate { id => "mutate_split_zeek_signatures_sub_message" + split => { "[zeek_cols][sub_message]" => "," } } + } } } else if ([log_source] == "sip") { @@ -2772,9 +2742,7 @@ filter { # sip.log # https://docs.zeek.org/en/stable/scripts/base/protocols/sip/main.zeek.html#type-SIP::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_sip" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2794,6 +2762,9 @@ filter { code => "event.set('[zeek_cols]', @zeek_sip_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_sip_commas" + split => { "[zeek_cols][request_path]" => "," + "[zeek_cols][response_path]" => "," } } } mutate { @@ -2809,9 +2780,7 @@ filter { # note that smb_cmd.referenced_file is exactly the same structure as the log line for smb_files. later on it will be # merged up as its own top-level entity so I don't have to duplicate the parsing effort below - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_smb_cmd" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2853,9 +2822,7 @@ filter { # smb_files.log # https://docs.zeek.org/en/stable/scripts/base/protocols/smb/main.zeek.html#type-SMB::FileInfo - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_smb_files_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2893,9 +2860,7 @@ filter { # smb_mapping.log # https://docs.zeek.org/en/stable/scripts/base/protocols/smb/main.zeek.html#type-SMB::TreeInfo - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_smb_mapping" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2930,9 +2895,7 @@ filter { # smtp.log # https://docs.zeek.org/en/stable/scripts/base/protocols/smtp/main.zeek.html#type-SMTP::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_smtp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2952,6 +2915,11 @@ filter { code => "event.set('[zeek_cols]', @zeek_smtp_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_smtp_commas" + split => { "[zeek_cols][rcptto]" => "," + "[zeek_cols][to]" => "," + "[zeek_cols][cc]" => "," + "[zeek_cols][path]" => "," } } } mutate { @@ -2967,9 +2935,7 @@ filter { # snmp.log # https://docs.zeek.org/en/stable/scripts/base/protocols/snmp/main.zeek.html#type-SNMP::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_snmp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3004,9 +2970,7 @@ filter { # socks.log # https://docs.zeek.org/en/stable/scripts/base/protocols/socks/main.zeek.html#type-SOCKS::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_socks" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3038,9 +3002,7 @@ filter { # software.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/software/main.zeek.html#type-Software::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_software" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3067,9 +3029,7 @@ filter { # wireguard.log # https://github.com/corelight/zeek-spicy-wireguard/blob/master/analyzer/main.zeek - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_wireguard" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3107,9 +3067,7 @@ filter { # ssh.log # https://docs.zeek.org/en/stable/scripts/base/protocols/ssh/main.zeek.html#type-SSH::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_ssh_with_all_fields_with_hassh" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3129,6 +3087,11 @@ filter { code => "event.set('[zeek_cols]', @zeek_ssh_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_ssh_commas" + split => { "[zeek_cols][hasshAlgorithms]" => "," + "[zeek_cols][hasshServerAlgorithms]" => "," + "[zeek_cols][cshka]" => "," + "[zeek_cols][sshka]" => "," } } } mutate { @@ -3144,9 +3107,7 @@ filter { # ssl.log # https://docs.zeek.org/en/stable/scripts/base/protocols/ssl/main.zeek.html#type-SSL::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_ssl_v1_with_ja3" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3166,6 +3127,11 @@ filter { code => "event.set('[zeek_cols]', @zeek_ssl_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_ssl_commas" + split => { "[zeek_cols][cert_chain_fuids]" => "," + "[zeek_cols][client_cert_chain_fuids]" => "," + "[zeek_cols][cert_chain_fps]" => "," + "[zeek_cols][client_cert_chain_fps]" => "," } } } mutate { @@ -3181,9 +3147,7 @@ filter { # stun.log # https://github.com/corelight/zeek-spicy-stun/blob/master/analyzer/main.zeek - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_stun" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3203,6 +3167,8 @@ filter { code => "event.set('[zeek_cols]', @zeek_stun_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_field_zeek_stun_attr_type" + split => { "[zeek_cols][attr_type]" => "," } } } mutate { @@ -3217,9 +3183,7 @@ filter { # stun.log # https://github.com/corelight/zeek-spicy-stun/blob/master/analyzer/main.zeek - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_stun_nat" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3253,9 +3217,7 @@ filter { # synchrophasor.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_synchrophasor" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3291,9 +3253,7 @@ filter { # synchrophasor_cmd.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_synchrophasor_cmd" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3328,9 +3288,7 @@ filter { # synchrophasor_cfg.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_synchrophasor_cfg" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3365,9 +3323,7 @@ filter { # synchrophasor_cfg_detail.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_synchrophasor_cfg_detail" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3402,9 +3358,7 @@ filter { # synchrophasor_data.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_synchrophasor_data" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3439,9 +3393,7 @@ filter { # synchrophasor_data_detail.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_synchrophasor_data_detail" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3476,9 +3428,7 @@ filter { # synchrophasor_hdr.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_synchrophasor_hdr" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3513,9 +3463,7 @@ filter { # syslog.log # https://docs.zeek.org/en/stable/scripts/base/protocols/syslog/main.zeek.html#type-Syslog::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_syslog" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3547,9 +3495,7 @@ filter { # tds.log # https://github.com/amzn/zeek-plugin-tds/blob/master/scripts/main.zeek - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_tds" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3584,9 +3530,7 @@ filter { # tds_rpc.log # https://github.com/amzn/zeek-plugin-tds/blob/master/scripts/main.zeek - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_tds_rpc" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3621,9 +3565,7 @@ filter { # tds_sql_batch.log # https://github.com/amzn/zeek-plugin-tds/blob/master/scripts/main.zeek - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_tds_sql_batch" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3658,9 +3600,7 @@ filter { # tftp.log # https://github.com/zeek/spicy-tftp - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_tftp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3695,9 +3635,7 @@ filter { # tunnel.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/tunnels/main.zeek.html#type-Tunnel::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_tunnel" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3738,9 +3676,7 @@ filter { # weird.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/notice/weird.zeek.html#type-Weird::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_weird" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3767,9 +3703,7 @@ filter { # x509.log # https://docs.zeek.org/en/stable/scripts/base/files/x509/main.zeek.html#type-X509::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_x509_v1" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3789,6 +3723,9 @@ filter { code => "event.set('[zeek_cols]', @zeek_x509_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_zeek_x509_san_ip" + split => { "[zeek][x509][san_ip]" => "," + "[zeek][x509][fingerprint]" => "," } } } mutate { id => "mutate_add_fields_zeek_x509" @@ -3801,9 +3738,7 @@ filter { # opcua_binary.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3839,9 +3774,7 @@ filter { # opcua_binary_activate_session.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_activate_session" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3877,9 +3810,7 @@ filter { # opcua_binary_activate_session_client_software_cert.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_activate_session_client_software_cert" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3915,9 +3846,7 @@ filter { # opcua_binary_activate_session_locale_id.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_activate_session_locale_id" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3953,9 +3882,7 @@ filter { # opcua_binary_aggregate_filter.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_aggregate_filter" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3991,9 +3918,7 @@ filter { # opcua_binary_event_filter_attribute_operand.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_event_filter_attribute_operand" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4029,9 +3954,7 @@ filter { # opcua_binary_event_filter_attribute_operand_browse_paths.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_event_filter_attribute_operand_browse_paths" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4067,9 +3990,7 @@ filter { # opcua_binary_browse.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_browse" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4105,9 +4026,7 @@ filter { # opcua_binary_browse_description.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_browse_description" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4143,9 +4062,7 @@ filter { # opcua_binary_browse_response_references.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_browse_response_references" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4181,9 +4098,7 @@ filter { # opcua_binary_browse_request_continuation_point.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_browse_request_continuation_point" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4219,9 +4134,7 @@ filter { # opcua_binary_browse_result.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_browse_result" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4257,9 +4170,7 @@ filter { # opcua_binary_close_session.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_close_session" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4295,9 +4206,7 @@ filter { # opcua_binary_event_filter_where_clause.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_event_filter_where_clause" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4333,9 +4242,7 @@ filter { # opcua_binary_event_filter_where_clause_elements.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_event_filter_where_clause_elements" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4371,9 +4278,7 @@ filter { # opcua_binary_create_monitored_items.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_create_monitored_items" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4409,9 +4314,7 @@ filter { # opcua_binary_create_monitored_items_create_item.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_create_monitored_items_create_item" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4447,9 +4350,7 @@ filter { # opcua_binary_create_session.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_create_session" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4485,9 +4386,7 @@ filter { # opcua_binary_create_session_discovery.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_create_session_discovery" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4523,9 +4422,7 @@ filter { # opcua_binary_create_session_endpoints.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_create_session_endpoints" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4561,9 +4458,7 @@ filter { # opcua_binary_create_session_user_token.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_create_session_user_token" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4599,9 +4494,7 @@ filter { # opcua_binary_create_subscription.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_create_subscription" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4637,9 +4530,7 @@ filter { # opcua_binary_data_change_filter.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_data_change_filter" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4675,9 +4566,7 @@ filter { # opcua_binary_diag_info_detail.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_diag_info_detail" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4713,9 +4602,7 @@ filter { # opcua_binary_event_filter_element_operand.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_event_filter_element_operand" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4751,9 +4638,7 @@ filter { # opcua_binary_event_filter.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_event_filter" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4789,9 +4674,7 @@ filter { # opcua_binary_get_endpoints.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_get_endpoints" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4827,9 +4710,7 @@ filter { # opcua_binary_get_endpoints_description.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_get_endpoints_description" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4865,9 +4746,7 @@ filter { # opcua_binary_get_endpoints_discovery.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_get_endpoints_discovery" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4903,9 +4782,7 @@ filter { # opcua_binary_get_endpoints_locale_id.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_get_endpoints_locale_id" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4941,9 +4818,7 @@ filter { # opcua_binary_get_endpoints_profile_uri.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_get_endpoints_profile_uri" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -4979,9 +4854,7 @@ filter { # opcua_binary_get_endpoints_user_token.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_get_endpoints_user_token" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5017,9 +4890,7 @@ filter { # opcua_binary_event_filter_literal_operand.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_event_filter_literal_operand" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5055,9 +4926,7 @@ filter { # opcua_binary_opensecure_channel.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_opensecure_channel" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5093,9 +4962,7 @@ filter { # opcua_binary_read.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_read" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5131,9 +4998,7 @@ filter { # opcua_binary_read_nodes_to_read.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_read_nodes_to_read" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5169,9 +5034,7 @@ filter { # opcua_binary_read_results.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_read_results" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5207,9 +5070,7 @@ filter { # opcua_binary_event_filter_select_clause.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_event_filter_select_clause" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5245,9 +5106,7 @@ filter { # opcua_binary_event_filter_simple_attribute_operand.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_event_filter_simple_attribute_operand" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5283,9 +5142,7 @@ filter { # opcua_binary_event_filter_simple_attribute_operand_browse_paths.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5321,9 +5178,7 @@ filter { # opcua_binary_status_code_detail.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_status_code_detail" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5359,9 +5214,7 @@ filter { # opcua_binary_variant_array_dims.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_variant_array_dims" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5397,9 +5250,7 @@ filter { # opcua_binary_variant_data.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_variant_data" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5435,9 +5286,7 @@ filter { # opcua_binary_variant_data_value.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_variant_data_value" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5473,9 +5322,7 @@ filter { # opcua_binary_variant_extension_object.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_variant_extension_object" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5511,9 +5358,7 @@ filter { # opcua_binary_variant_metadata.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_opcua_binary_variant_metadata" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5547,9 +5392,7 @@ filter { } else { # some other unknown zeek opcua- log file. should start with ts at least! - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { csv { id => "csv_zeek_unknown_opcua" columns => ["ts"] @@ -5579,9 +5422,7 @@ filter { # Zeek Logging analyzer confirmations and violations into analyzer.log # https://docs.zeek.org/en/master/scripts/base/frameworks/analyzer/logging.zeek.html - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_diagnostic_analyzer" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5611,9 +5452,7 @@ filter { # broker.log # https://docs.zeek.org/en/master/scripts/base/frameworks/broker/log.zeek.html - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_diagnostic_broker" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5644,9 +5483,7 @@ filter { # Reports analysis of missing traffic. Zeek bases its conclusions on analysis of TCP sequence numbers. # https://docs.zeek.org/en/master/logs/capture-loss-and-reporter.html - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_diagnostic_capture_loss" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5677,9 +5514,7 @@ filter { # Logging for establishing and controlling a cluster of Zeek instances # https://docs.zeek.org/en/master/scripts/base/frameworks/cluster/main.zeek.html#type-Cluster::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_diagnostic_cluster" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5710,9 +5545,7 @@ filter { # Logging for Zeek configuration changes # https://docs.zeek.org/en/master/scripts/base/frameworks/config/main.zeek.html#type-Config::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_diagnostic_config" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5742,9 +5575,7 @@ filter { # packet_filter.log # https://docs.zeek.org/en/master/scripts/base/frameworks/packet-filter/main.zeek.html#type-PacketFilter::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_diagnostic_packet_filter" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5774,9 +5605,7 @@ filter { # print.log # https://docs.zeek.org/en/master/scripts/base/frameworks/logging/main.zeek.html#type-Log::PrintLogInfo - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_diagnostic_print" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5796,11 +5625,10 @@ filter { code => "event.set('[zeek_cols]', @zeek_diagnostic_print_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "split_zeek_diagnostic_print_vals" + split => { "[zeek_cols][vals]" => "," } } } - mutate { id => "split_zeek_diagnostic_print_vals" - split => { "[zeek_cols][vals]" => "," } } - mutate { id => "mutate_add_tag_zeek_diagnostic_print" add_tag => [ "_zeekdiagnostic" ] } @@ -5810,9 +5638,7 @@ filter { # reporter.log # https://docs.zeek.org/en/master/scripts/base/frameworks/reporter/main.zeek.html#type-Reporter::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_diagnostic_reporter" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5842,9 +5668,7 @@ filter { # stats.log # https://docs.zeek.org/en/master/scripts/policy/misc/stats.zeek.html#type-Stats::Info - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { dissect { id => "dissect_zeek_diagnostic_stats" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5871,9 +5695,7 @@ filter { } else { - if "_jsonparsesuccess" in [tags] { - - } else { + if ("_jsonparsesuccess" not in [tags]) { # some other unknown zeek log file. should start with ts at least! csv { id => "csv_zeek_unknown" diff --git a/logstash/pipelines/zeek/12_zeek_mutate.conf b/logstash/pipelines/zeek/12_zeek_mutate.conf index a56866eee..5cf0586e8 100644 --- a/logstash/pipelines/zeek/12_zeek_mutate.conf +++ b/logstash/pipelines/zeek/12_zeek_mutate.conf @@ -67,14 +67,13 @@ filter { base64encode => true } - if ([source][mac]) { - mutate { id => "mutate_split_zeek_source_mac" - split => { "[source][mac]" => "," } } - } - - if ([destination][mac]) { - mutate { id => "mutate_split_zeek_destination_mac" - split => { "[destination][mac]" => "," } } + if ("_jsonparsesuccess" not in [tags]) { + mutate { id => "mutate_split_zeek_commas_post_parse" + split => { "[source][mac]" => "," + "[destination][mac]" => "," + "[zeek][fuid]" => "," + "[related][user]" => "," + "[network][protocol]" => "," } } } if ([zeek][uid]) { @@ -87,10 +86,6 @@ filter { merge => { "[event][id]" => "[zeek][uid]" } } } - # if present, FUIDs are always a vector (comma-separated) - if ([zeek][fuid]) { mutate { id => "mutate_split_zeek_fuids" - split => { "[zeek][fuid]" => "," } } } - if ([zeek][ts]) { if ([zeek][ts] == "0.000000") { @@ -140,10 +135,6 @@ filter { } # if ([zeek][ts]) # set user and transport- and application-level protocols if specified - if ([related][user]) { - mutate { id => "mutate_split_related_user" - split => { "[related][user]" => "," } } - } if ([network][transport]) { translate { id => "translate_zeek_proto" @@ -151,17 +142,14 @@ filter { target => "[ipProtocol]" dictionary_path => "/etc/ip_protocol_name_to_number.yaml" } - mutate { id => "mutate_split_zeek_proto" - split => { "[network][transport]" => "," } } + if ("_jsonparsesuccess" not in [tags]) { + mutate { id => "mutate_split_zeek_proto" + split => { "[network][transport]" => "," } } + } mutate { id => "mutate_merge_zeek_proto" merge => { "[protocol]" => "[network][transport]" } } } - if ([network][protocol]) { - mutate { id => "mutate_split_zeek_service" - split => { "[network][protocol]" => "," } } - } - if ([log_source] == "conn") { ############################################################################################################################# # conn.log specific logic @@ -174,9 +162,6 @@ filter { # TODO: can we normalize this with suricata.flow.state somewhere? } - mutate { id => "mutate_split_zeek_conn_tunnel_parents" - split => { "[zeek][conn][tunnel_parents]" => "," } } - if ([zeek][conn][vlan] or [zeek][conn][inner_vlan]) { # ECS - zeek.conn.vlan -> network.vlan.id if ([zeek][conn][vlan]) { @@ -261,8 +246,15 @@ filter { id => "ruby_bsap_ip_rdb_variables" init => "@bsap_ip_rdb_fields = [ :var, :val ]" code => " - vars = event.get('[zeek][bsap_ip_rdb][variables]').to_s.split(',').zip( - event.get('[zeek][bsap_ip_rdb][variable_value]').to_s.split(',')).map{ |x| @bsap_ip_rdb_fields.zip(x).to_h } + vars = event.get('[zeek][bsap_ip_rdb][variables]') + values = event.get('[zeek][bsap_ip_rdb][variable_value]') + if !vars.is_a?(Array) then + vars = vars.to_s.split(',') + end + if !values.is_a?(Array) then + values = values.to_s.split(',') + end + vars = vars.zip(values).map{ |x| @bsap_ip_rdb_fields.zip(x).to_h } event.set('[zeek][bsap_ip_rdb][variables]', vars) event.set('[zeek][bsap_ip_rdb][variable_count]', vars.length) " @@ -280,8 +272,15 @@ filter { id => "ruby_bsap_serial_rdb_variables" init => "@bsap_serial_rdb_fields = [ :var, :val ]" code => " - vars = event.get('[zeek][bsap_serial_rdb][variables]').to_s.split(',').zip( - event.get('[zeek][bsap_serial_rdb][variable_value]').to_s.split(',')).map{ |x| @bsap_serial_rdb_fields.zip(x).to_h } + vars = event.get('[zeek][bsap_serial_rdb][variables]') + values = event.get('[zeek][bsap_serial_rdb][variable_value]') + if !vars.is_a?(Array) then + vars = vars.to_s.split(',') + end + if !values.is_a?(Array) then + values = values.to_s.split(',') + end + vars = vars.zip(values).map{ |x| @bsap_serial_rdb_fields.zip(x).to_h } event.set('[zeek][bsap_serial_rdb][variables]', vars) event.set('[zeek][bsap_serial_rdb][variable_count]', vars.length) " @@ -333,8 +332,6 @@ filter { } if ([zeek][dhcp][msg_types]) { - mutate { id => "mutate_split_zeek_dhcp_msg_types" - split => { "[zeek][dhcp][msg_types]" => "," } } ruby { id => "ruby_zeek_dhcp_msg_types" path => "/usr/share/logstash/malcolm-ruby/make_unique_array.rb" @@ -401,21 +398,20 @@ filter { ############################################################################################################################# # dns.log specific logic - mutate { id => "mutate_split_zeek_dns_TTLs" - split => { "[zeek][dns][TTLs]" => "," } } - # this must be done because [dns][ip] only handles IPv4 or IPv6 addresses # but [answers] can contain hostnames as well if ([zeek][dns][answers]) { ruby { id => "ruby_zeek_dns_answers_ip_extract" # todo: adjust this regex so it at least sort of catches IPv6 as well - code => "event.set('[@metadata][answers_ip]', event.get('[zeek][dns][answers]').scan(/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/).join(','))" + code => " + answers = event.get('[zeek][dns][answers]') + if answers.is_a?(Array) then + answers = answers.join(',') + end + event.set('[@metadata][answers_ip]', answers.scan(/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/)) + " } - mutate { id => "mutate_split_zeek_dns_answers" - split => { "[zeek][dns][answers]" => "," } } - mutate { id => "mutate_split_zeek_dns_answers_ip" - split => { "[@metadata][answers_ip]" => "," } } # ECS - zeek.dns -> dns.answers ruby { id => "ruby_zeek_dns_answers_to_ecs" @@ -521,12 +517,6 @@ filter { } } - if ([zeek][files][parent_fuid]) { mutate { id => "mutate_split_zeek_files_parent_fuid" - split => { "[zeek][files][parent_fuid]" => "," } } } - - if ([zeek][files][analyzers]) { mutate { id => "mutate_split_zeek_files_analyzers" - split => { "[zeek][files][analyzers]" => "," } } } - if ([zeek][files][source]) { # do some normalization on files source @@ -589,7 +579,11 @@ filter { id => "ruby_genisys_payload" init => "@genisys_fields = [ :address, :data ]" code => " - vars = event.get('[zeek][genisys][payload_raw]').to_s.split(',').map{ |x| x.split('=') }.map{ |x| @genisys_fields.zip(x).to_h } + vars = event.get('[zeek][genisys][payload_raw]') + if !vars.is_a?(Array) then + vars = vars.to_s.split(',') + end + vars = vars.map{ |x| x.split('=') }.map{ |x| @genisys_fields.zip(x).to_h } event.set('[zeek][genisys][payload]', vars) unless vars.nil? or (vars.length == 0) " } @@ -622,21 +616,6 @@ filter { ############################################################################################################################# # http.log specific logic - mutate { id => "mutate_split_zeek_http_orig_fuids" - split => { "[zeek][http][orig_fuids]" => "," } } - mutate { id => "mutate_split_zeek_http_orig_filenames" - split => { "[zeek][http][orig_filenames]" => "," } } - mutate { id => "mutate_split_zeek_http_orig_mime_types" - split => { "[zeek][http][orig_mime_types]" => "," } } - mutate { id => "mutate_split_zeek_http_resp_fuids" - split => { "[zeek][http][resp_fuids]" => "," } } - mutate { id => "mutate_split_zeek_http_resp_filenames" - split => { "[zeek][http][resp_filenames]" => "," } } - mutate { id => "mutate_split_zeek_http_resp_mime_types" - split => { "[zeek][http][resp_mime_types]" => "," } } - mutate { id => "mutate_split_zeek_http_proxied" - split => { "[zeek][http][proxied]" => "," } } - # translate Zeek's double-backslashes single backslash in filenames ruby { id => "ruby_zeek_http_orig_filenames_dedoubleslash" @@ -657,8 +636,6 @@ filter { if ([zeek][http][tags]) { # merge zeek.http.tags into tags and then remove original - mutate { id => "mutate_split_zeek_http_tags" - split => { "[zeek][http][tags]" => "," } } mutate { id => "mutate_merge_zeek_http_tags" merge => { "[tags]" => "[zeek][http][tags]" } } mutate { id => "mutate_remove_field_zeek_http_tags" @@ -797,21 +774,6 @@ filter { ############################################################################################################################# # ipsec.log specific logic - mutate { id => "mutate_split_zeek_ipsec_vendor_ids" - split => { "[zeek][ipsec][vendor_ids]" => "," } } - mutate { id => "mutate_split_zeek_ipsec_notify_messages" - split => { "[zeek][ipsec][notify_messages]" => "," } } - mutate { id => "mutate_split_zeek_ipsec_transforms" - split => { "[zeek][ipsec][transforms]" => "," } } - mutate { id => "mutate_split_zeek_ipsec_ke_dh_groups" - split => { "[zeek][ipsec][ke_dh_groups]" => "," } } - mutate { id => "mutate_split_zeek_ipsec_proposals" - split => { "[zeek][ipsec][proposals]" => "," } } - mutate { id => "mutate_split_zeek_ipsec_certificates" - split => { "[zeek][ipsec][certificates]" => "," } } - mutate { id => "mutate_split_zeek_ipsec_transform_attributes" - split => { "[zeek][ipsec][transform_attributes]" => "," } } - ruby { id => "ruby_zeek_field_ipsec_flags" code => ' @@ -862,15 +824,6 @@ filter { ############################################################################################################################# # intel.log specific logic - if ([zeek][intel][sources]) { mutate { id => "mutate_split_zeek_intel_sources" - split => { "[zeek][intel][sources]" => "," } } } - - if ([zeek][intel][matched]) { mutate { id => "mutate_split_zeek_intel_matched" - split => { "[zeek][intel][matched]" => "," } } } - - if ([zeek][intel][cif_tags]) { mutate { id => "mutate_split_zeek_intel_cif_tags" - split => { "[zeek][intel][cif_tags]" => "," } } } - if ([zeek][intel][sources]) or ([zeek][intel][cif_tags]) { ruby { id => "ruby_zeek_intel_sources_and_tags_comma_fix" @@ -906,23 +859,10 @@ filter { merge => { "[krb5][sname]" => "[zeek][kerberos][sname]" } } } - if ([zeek][kerberos][client_cert_fuid]) { mutate { id => "mutate_split_zeek_kerberos_client_cert_fuid" - split => { "[zeek][kerberos][client_cert_fuid]" => "," } } } - - if ([zeek][kerberos][server_cert_fuid]) { mutate { id => "mutate_split_zeek_kerberos_server_cert_fuid" - split => { "[zeek][kerberos][server_cert_fuid]" => "," } } } - } else if ([log_source] == "ldap") { ############################################################################################################################# # ldap.log specific logic - mutate { id => "mutate_split_zeek_ldap_fields" - split => { "[zeek][ldap][operation]" => "," - "[zeek][ldap][result_code]" => "," - "[zeek][ldap][result_message]" => "," - "[zeek][ldap][object]" => "," - "[zeek][ldap][argument]" => "," } } - if ([zeek][ldap][object]) or ([zeek][ldap][argument]) { ruby { id => "ruby_zeek_ldap_object_and_arguments_adjustment" @@ -972,14 +912,6 @@ filter { ############################################################################################################################# # ldap_search.log specific logic - mutate { id => "mutate_split_zeek_ldap_search_fields" - split => { "[zeek][ldap_search][scope]" => "," - "[zeek][ldap_search][deref]" => "," - "[zeek][ldap_search][attributes]" => "," - "[zeek][ldap_search][base_object]" => "," - "[zeek][ldap_search][result_code]" => "," - "[zeek][ldap_search][result_message]" => "," } } - if ([zeek][ldap_search][base_object]) { ruby { id => "ruby_zeek_ldap_search_base_object_adjustment" @@ -1015,11 +947,6 @@ filter { ############################################################################################################################# # modbus_detailed.log specific logic - if ([zeek][modbus_detailed][values]) { - mutate { id => "mutate_split_zeek_modbus_detailed_values" - split => { "[zeek][modbus_detailed][values]" => "," } } - } - # rename some fields to make correlating modbus easier between logs mutate { id => "mutate_rename_modbus_detailed_fields" @@ -1061,16 +988,6 @@ filter { ############################################################################################################################# # modbus_read_write_multiple_registers.log specific logic - if ([zeek][modbus_read_write_multiple_registers][read_registers]) { - mutate { id => "mutate_split_zeek_modbus_read_write_multiple_registers_read_registers" - split => { "[zeek][modbus_read_write_multiple_registers][read_registers]" => "," } } - } - - if ([zeek][modbus_read_write_multiple_registers][write_registers]) { - mutate { id => "mutate_split_zeek_modbus_read_write_multiple_registers_write_registers" - split => { "[zeek][modbus_read_write_multiple_registers][write_registers]" => "," } } - } - # rename some fields to make correlating modbus easier between logs mutate { id => "mutate_rename_modbus_read_write_multiple_registers_fields" @@ -1095,14 +1012,6 @@ filter { add_field => { "[client][bytes]" => "%{[zeek][mqtt_publish][payload_len]}" } } } - } else if ([log_source] == "mqtt_subscribe") { - ############################################################################################################################# - # mqtt_subscribe.log specific logic - - mutate { id => "mutate_split_zeek_mqtt_subscribe_fields" - split => { "[zeek][mqtt_subscribe][topics]" => "," - "[zeek][mqtt_subscribe][qos_levels]" => "," } } - } else if ([log_source] == "mysql") { ############################################################################################################################# # mysql.log specific logic @@ -1116,9 +1025,6 @@ filter { ############################################################################################################################# # notice.log specific logic - mutate { id => "mutate_split_zeek_notice_actions" - split => { "[zeek][notice][actions]" => "," } } - ruby { id => "ruby_add_field_zeek_notice_event_original" code => " event.set('[event][original]', [event.get('[zeek][notice][msg]'), @@ -1474,32 +1380,6 @@ filter { add_field => { "[zeek][software][software_type]" => "OPCUA" } } } - } else if ([log_source] == "ospf") { - ############################################################################################################################# - # ospf.log specific logic - - mutate { id => "mutate_split_zeek_ospf_neighbors" - split => { "[zeek][ospf][neighbors]" => "," } } - mutate { id => "mutate_split_zeek_ospf_routers" - split => { "[zeek][ospf][routers]" => "," } } - mutate { id => "mutate_split_zeek_ospf_metrics" - split => { "[zeek][ospf][metrics]" => "," } } - mutate { id => "mutate_split_zeek_ospf_fwd_addrs" - split => { "[zeek][ospf][fwd_addrs]" => "," } } - mutate { id => "mutate_split_zeek_ospf_route_tags" - split => { "[zeek][ospf][route_tags]" => "," } } - mutate { id => "mutate_split_zeek_link_prefixes" - split => { "[zeek][ospf][link_prefixes]" => "," } } - mutate { id => "mutate_split_zeek_intra_prefixes" - split => { "[zeek][ospf][intra_prefixes]" => "," } } - - } else if ([log_source] == "pe") { - ############################################################################################################################# - # pe.log specific logic - - mutate { id => "mutate_split_zeek_pe_section_names" - split => { "[zeek][pe][section_names]" => "," } } - } else if ([log_source] == "radius") { ############################################################################################################################# # radius.log specific logic @@ -1526,13 +1406,6 @@ filter { } } - } else if ([log_source] == "rdp") { - ############################################################################################################################# - # rdp.log specific logic - - mutate { id => "mutate_split_zeek_rdp_client_channels" - split => { "[zeek][rdp][client_channels]" => "," } } - } else if ([log_source] == "s7comm_read_szl") { ############################################################################################################################# # s7comm_read_szl.log specific logic @@ -1560,8 +1433,6 @@ filter { # _carved signature logs' sub_message contains fuid(s) comma-separated if ([zeek][signatures][sub_message]) { - mutate { id => "mutate_split_zeek_signatures_sub_message" - split => { "[zeek][signatures][sub_message]" => "," } } mutate { id => "mutate_rename_zeek_signatures_sub_message" rename => { "[zeek][signatures][sub_message]" => "[zeek][fuid]" } } } @@ -1714,12 +1585,6 @@ filter { event.set("[zeek][sip][version]", versions.uniq)' } - mutate { id => "mutate_split_zeek_sip_request_path" - split => { "[zeek][sip][request_path]" => "," } } - - mutate { id => "mutate_split_zeek_sip_response_path" - split => { "[zeek][sip][response_path]" => "," } } - if ([zeek][sip][request_body_len]) { mutate { id => "mutate_add_field_zeek_sip_client_bytes" add_field => { "[client][bytes]" => "%{[zeek][sip][request_body_len]}" } } } @@ -1886,15 +1751,6 @@ filter { ############################################################################################################################# # smtp.log specific logic - mutate { id => "mutate_split_zeek_smtp_rcptto" - split => { "[zeek][smtp][rcptto]" => "," } } - mutate { id => "mutate_split_zeek_smtp_to" - split => { "[zeek][smtp][to]" => "," } } - mutate { id => "mutate_split_zeek_smtp_cc" - split => { "[zeek][smtp][cc]" => "," } } - mutate { id => "mutate_split_zeek_smtp_path" - split => { "[zeek][smtp][path]" => "," } } - if ([zeek][smtp][user_agent]) { mutate { id => "mutate_merge_zeek_smtp_user_agent" merge => { "[email][useragent]" => "[zeek][smtp][user_agent]" } } @@ -1975,18 +1831,6 @@ filter { merge => { "[ssh][hasshServer]" => "[zeek][ssh][hasshServer]" } } } - if ([zeek][ssh][hasshAlgorithms]) { mutate { id => "mutate_split_zeek_ssh_hasshAlgorithms" - split => { "[zeek][ssh][hasshAlgorithms]" => "," } } } - - if ([zeek][ssh][hasshServerAlgorithms]) { mutate { id => "mutate_split_zeek_ssh_hasshServerAlgorithms" - split => { "[zeek][ssh][hasshServerAlgorithms]" => "," } } } - - if ([zeek][ssh][cshka]) { mutate { id => "mutate_split_zeek_ssh_cshka" - split => { "[zeek][ssh][cshka]" => "," } } } - - if ([zeek][ssh][sshka]) { mutate { id => "mutate_split_zeek_ssh_sshka" - split => { "[zeek][ssh][sshka]" => "," } } } - } else if ([log_source] == "ssl") { ############################################################################################################################# @@ -2098,15 +1942,7 @@ filter { merge => { "[tls][ja3s]" => "[zeek][ssl][ja3s]" } } } - if ([zeek][ssl][cert_chain_fuids]) { mutate { id => "mutate_split_zeek_ssl_cert_chain_fuids" - split => { "[zeek][ssl][cert_chain_fuids]" => "," } } } - - if ([zeek][ssl][client_cert_chain_fuids]) { mutate { id => "mutate_split_zeek_ssl_client_cert_chain_fuids" - split => { "[zeek][ssl][client_cert_chain_fuids]" => "," } } } - if ([zeek][ssl][cert_chain_fps]) { - mutate { id => "mutate_split_zeek_ssl_cert_chain_fps" - split => { "[zeek][ssl][cert_chain_fps]" => "," } } # store certificate chain fingerprints in zeek_x509.fingerprint as well for cross-reference with x509.log mutate { id => "mutate_merge_zeek_ssl_cert_chain_fps" merge => { "[zeek][x509][fingerprint]" => "[zeek][ssl][cert_chain_fps]" } } @@ -2116,8 +1952,6 @@ filter { } if ([zeek][ssl][client_cert_chain_fps]) { - mutate { id => "mutate_split_zeek_ssl_client_cert_chain_fps" - split => { "[zeek][ssl][client_cert_chain_fps]" => "," } } # store certificate chain fingerprints in zeek_x509.fingerprint as well for cross-reference with x509.log mutate { id => "mutate_merge_zeek_ssl_client_cert_chain_fps" merge => { "[zeek][x509][fingerprint]" => "[zeek][ssl][client_cert_chain_fps]" } } @@ -2191,15 +2025,6 @@ filter { if ([zeek][ssl][subject_full]) { mutate { id => "mutate_add_field_ecs_zeek_tls_subject_full" add_field => { "[tls][server][subject]" => "%{[zeek][ssl][subject_full]}" } } } - } else if ([log_source] == "stun") { - ############################################################################################################################# - # stun.log specific logic - - if ([zeek][stun][attr_type]) { - mutate { id => "mutate_split_field_zeek_stun_attr_type" - split => { "[zeek][stun][attr_type]" => "," } } - } - } else if ([log_source] == "stun_nat") { ############################################################################################################################# # stun_nat.log specific logic @@ -2235,42 +2060,44 @@ filter { rename => { "[zeek][synchrophasor_hdr][header_time_stamp]" => "[zeek][synchrophasor][header_time_stamp]" } } - mutate { - id => "mutate_split_synchrophasor_fields" - split => { "[zeek][synchrophasor_cfg_detail][analog_conv_analog_flags]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][analog_conv_mag_scale]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][analog_conv_offset]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][analog_conv_user_defined_scaling]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][annam]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][dgnam]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][digital_conv_normal_status_mask]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][digital_conv_valid_inputs_mask]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_angle_adj]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_calibration_mag_adj]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_calibration_phas_adj]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_downsampled_fir_filter]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_downsampled_no_fir_filter]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_downsampled_reselection]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_filtered_without_changing_sampling]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_mod_appl]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_phasor_component]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_phasor_type]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_phunit]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_phvalue]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_pseudo_phasor_val]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_rotation_phase_adj]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_scale_factor]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_upsampled_extrapolation]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_upsampled_interpolation]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_user_def]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phnam]" => "," } - split => { "[zeek][synchrophasor_cmd][extframe]" => "," } - split => { "[zeek][synchrophasor_data_detail][analog_data]" => "," } - split => { "[zeek][synchrophasor_data_detail][digital]" => "," } - split => { "[zeek][synchrophasor_data_detail][est_polar_angle]" => "," } - split => { "[zeek][synchrophasor_data_detail][est_polar_magnitude]" => "," } - split => { "[zeek][synchrophasor_data_detail][est_rectangular_imaginary]" => "," } - split => { "[zeek][synchrophasor_data_detail][est_rectangular_real]" => "," } + if ("_jsonparsesuccess" not in [tags]) { + mutate { + id => "mutate_split_synchrophasor_fields" + split => { "[zeek][synchrophasor_cfg_detail][analog_conv_analog_flags]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][analog_conv_mag_scale]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][analog_conv_offset]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][analog_conv_user_defined_scaling]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][annam]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][dgnam]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][digital_conv_normal_status_mask]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][digital_conv_valid_inputs_mask]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_angle_adj]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_calibration_mag_adj]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_calibration_phas_adj]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_downsampled_fir_filter]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_downsampled_no_fir_filter]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_downsampled_reselection]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_filtered_without_changing_sampling]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_mod_appl]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_phasor_component]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_phasor_type]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_phunit]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_phvalue]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_pseudo_phasor_val]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_rotation_phase_adj]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_scale_factor]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_upsampled_extrapolation]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_upsampled_interpolation]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_user_def]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phnam]" => "," } + split => { "[zeek][synchrophasor_cmd][extframe]" => "," } + split => { "[zeek][synchrophasor_data_detail][analog_data]" => "," } + split => { "[zeek][synchrophasor_data_detail][digital]" => "," } + split => { "[zeek][synchrophasor_data_detail][est_polar_angle]" => "," } + split => { "[zeek][synchrophasor_data_detail][est_polar_magnitude]" => "," } + split => { "[zeek][synchrophasor_data_detail][est_rectangular_imaginary]" => "," } + split => { "[zeek][synchrophasor_data_detail][est_rectangular_real]" => "," } + } } if ([zeek][synchrophasor][frame_size]) { @@ -2290,15 +2117,20 @@ filter { ruby { id => "ruby_zeek_tds_rpc_parse_parameter" code => " - parameterHash = event.get('[zeek][tds_rpc][parameter]').split(',').each_with_object({}) do |a, hash| + params = event.get('[zeek][tds_rpc][parameter]') + if !params.is_a?(Array) then + params = params.to_s.split(',') + end + parameterHash = params.each_with_object({}) do |a, hash| key,value = a.split(/\s*=\s*/) hash[key.to_s.delete_prefix('@')] = value end event.set('[zeek][tds_rpc][parameters]', parameterHash.reject{|k,v| k.nil? || v.nil? || k.empty? || v.empty?})" } - - mutate { id => "mutate_split_field_tds_rpc_parameter" - split => { "[zeek][tds_rpc][parameter]" => "," } } + if ("_jsonparsesuccess" not in [tags]) { + mutate { id => "mutate_split_field_tds_rpc_parameter" + split => { "[zeek][tds_rpc][parameter]" => "," } } + } } } else if ([log_source] == "weird") { @@ -2433,12 +2265,6 @@ filter { } } - if ([zeek][x509][san_ip]) { mutate { id => "mutate_split_zeek_x509_san_ip" - split => { "[zeek][x509][san_ip]" => "," } } } - - if ([zeek][x509][fingerprint]) { mutate { id => "mutate_split_zeek_x509_fingerprint" - split => { "[zeek][x509][fingerprint]" => "," } } } - # TODO: ECS TLS/X509 nestings? # - https://www.elastic.co/guide/en/ecs/current/ecs-tls.html#ecs-tls-nestings # - https://www.elastic.co/guide/en/ecs/current/ecs-x509.html#_field_reuse_27 From 02898f8c1b658e2d4e5ba6d6f8e955bc283b3d5c Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 20 Feb 2024 13:12:52 -0700 Subject: [PATCH 08/79] for supporting JSON logs from Zeek (idaholab/Malcolm#65); almost certainly broken at this point --- logstash/pipelines/zeek/12_zeek_mutate.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash/pipelines/zeek/12_zeek_mutate.conf b/logstash/pipelines/zeek/12_zeek_mutate.conf index 5cf0586e8..321246729 100644 --- a/logstash/pipelines/zeek/12_zeek_mutate.conf +++ b/logstash/pipelines/zeek/12_zeek_mutate.conf @@ -607,7 +607,7 @@ filter { if ([zeek][gquic][version]) { ruby { id => "ruby_zeek_set_quic_version" - code => "event.set('[quic][version]', 'Q' + event.get('[zeek][gquic][version]').rjust(3, '0'))" + code => "event.set('[quic][version]', 'Q' + event.get('[zeek][gquic][version]').to_s.rjust(3, '0'))" } } From 3945ba46f08e2cdb0774b71ce0e6041e9a1cdb06 Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 20 Feb 2024 13:14:56 -0700 Subject: [PATCH 09/79] bump arkime to v5.0.1 (https://github.com/arkime/arkime/blob/05441d6192bbbdab71ca242b2d9dc62196324dc7/CHANGELOG#L33-L55) --- Dockerfiles/arkime.Dockerfile | 2 +- hedgehog-iso/arkime/Dockerfile | 2 +- hedgehog-raspi/sensor_install.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index cdc34e593..ac8d1f776 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -7,7 +7,7 @@ ENV TERM xterm ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 -ENV ARKIME_VERSION "v5.0.0" +ENV ARKIME_VERSION "v5.0.1" ENV ARKIME_DIR "/opt/arkime" ENV ARKIME_URL "https://github.com/arkime/arkime.git" ENV ARKIME_LOCALELASTICSEARCH no diff --git a/hedgehog-iso/arkime/Dockerfile b/hedgehog-iso/arkime/Dockerfile index 53a312ff0..4ef89f0da 100644 --- a/hedgehog-iso/arkime/Dockerfile +++ b/hedgehog-iso/arkime/Dockerfile @@ -6,7 +6,7 @@ LABEL maintainer="malcolm@inl.gov" ENV DEBIAN_FRONTEND noninteractive -ENV ARKIME_VERSION "5.0.0" +ENV ARKIME_VERSION "5.0.1" ENV ARKIME_DIR "/opt/arkime" RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sources && \ diff --git a/hedgehog-raspi/sensor_install.sh b/hedgehog-raspi/sensor_install.sh index 799a326de..d15225bcd 100644 --- a/hedgehog-raspi/sensor_install.sh +++ b/hedgehog-raspi/sensor_install.sh @@ -58,7 +58,7 @@ BUILD_ERROR_CODE=1 build_arkime(){ mkdir -p /tmp/arkime-deb - arkime_ver='5.0.0-1' + arkime_ver='5.0.1-1' curl -sSL -o /tmp/arkime-deb/arkime.deb "https://github.com/arkime/arkime/releases/download/v5.0.0/arkime_${arkime_ver}.ubuntu2204_arm64.deb" dpkg -i /tmp/arkime-deb/*.deb || apt-get -f install -y --no-install-suggests } @@ -66,7 +66,7 @@ build_arkime(){ build_arkime_src(){ arkime_repo='https://github.com/arkime/arkime.git' - arkime_ver='5.0.0' + arkime_ver='5.0.1' arkime_dir='/opt/arkime' build_jobs=$((PROC_CNT/2)) From d24e1bea6def8faacc5b1f8fda6b7d61b19055eb Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 20 Feb 2024 14:38:30 -0700 Subject: [PATCH 10/79] fix idaholab/Malcolm#428, hard-coded date value in Kibana pivot links --- nginx/nginx_idark2dash_rewrite_kibana.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/nginx_idark2dash_rewrite_kibana.conf b/nginx/nginx_idark2dash_rewrite_kibana.conf index 47ee989a4..2904b0734 100644 --- a/nginx/nginx_idark2dash_rewrite_kibana.conf +++ b/nginx/nginx_idark2dash_rewrite_kibana.conf @@ -1 +1 @@ -rewrite ^.*/idark2dash/(.*) $dashboards_proxy_url/app/dashboards#/view/0ad3d7c2-3441-485e-9dfe-dbb22e84e576?_g=(refreshInterval:(pause:!t,value:60000),time:(from:'2024-02-01T15:45:45.793Z',to:'2024-02-06T16:00:50.775Z'))&_a=(filters:!((meta:(alias:!n,disabled:!f,index:'$sessions_index',key:$filter_field,negate:!f,params:(query:'$filter_value'),type:phrase),query:(match_phrase:($filter_field:'$filter_value')))))? redirect; \ No newline at end of file +rewrite ^.*/idark2dash/(.*) $dashboards_proxy_url/app/dashboards#/view/0ad3d7c2-3441-485e-9dfe-dbb22e84e576?_g=(refreshInterval:(pause:!t,value:0),time:(from:$filter_start_time,to:$filter_stop_time))&_a=(filters:!((meta:(alias:!n,disabled:!f,index:'$sessions_index',key:$filter_field,negate:!f,params:(query:'$filter_value'),type:phrase),query:(match_phrase:($filter_field:'$filter_value')))))? redirect; \ No newline at end of file From 88045007c30ca89a807831592d305e45e525ec21 Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 20 Feb 2024 15:09:58 -0700 Subject: [PATCH 11/79] fix idaholab/Malcolm#427, links on landing page should open in a new tab --- file-monitor/scripts/extracted_files_http_server.py | 1 + nginx/landingpage/index.html | 1 + 2 files changed, 2 insertions(+) diff --git a/file-monitor/scripts/extracted_files_http_server.py b/file-monitor/scripts/extracted_files_http_server.py index 48ef609e7..2f10d9a97 100755 --- a/file-monitor/scripts/extracted_files_http_server.py +++ b/file-monitor/scripts/extracted_files_http_server.py @@ -103,6 +103,7 @@ def do_GET(self): # with doc.head: + base(target="_blank") meta(charset="utf-8") meta(name="viewport", content="width=device-width, initial-scale=1, shrink-to-fit=no") link(rel="icon", href=f"{args.assetsDirRespReplacer}favicon.ico", type="image/x-icon") diff --git a/nginx/landingpage/index.html b/nginx/landingpage/index.html index 67b62c54c..308821615 100644 --- a/nginx/landingpage/index.html +++ b/nginx/landingpage/index.html @@ -2,6 +2,7 @@ + From 276ee807571ec282f914cf5f316731f6a9738a58 Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 21 Feb 2024 07:16:43 -0700 Subject: [PATCH 12/79] bump opensearch and dashboards to v2.12.0 (https://github.com/opensearch-project/opensearch-build/blob/main/release-notes/opensearch-release-notes-2.12.0.md) --- Dockerfiles/dashboards.Dockerfile | 8 ++++---- Dockerfiles/opensearch.Dockerfile | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfiles/dashboards.Dockerfile b/Dockerfiles/dashboards.Dockerfile index b4e6e828d..de1679bf4 100644 --- a/Dockerfiles/dashboards.Dockerfile +++ b/Dockerfiles/dashboards.Dockerfile @@ -1,4 +1,4 @@ -FROM opensearchproject/opensearch-dashboards:2.11.1 +FROM opensearchproject/opensearch-dashboards:2.12.0 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' @@ -20,7 +20,7 @@ ENV PUSER_PRIV_DROP true ENV TERM xterm ENV TINI_VERSION v0.19.0 -ENV OSD_TRANSFORM_VIS_VERSION 2.11.0 +ENV OSD_TRANSFORM_VIS_VERSION 2.11.1 ARG NODE_OPTIONS="--max_old_space_size=4096" ENV NODE_OPTIONS $NODE_OPTIONS @@ -40,8 +40,8 @@ RUN yum upgrade -y && \ /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityDashboards --allow-root && \ cd /tmp && \ unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ - sed -i "s/2\.11\.0/2\.11\.1/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \ - sed -i "s/2\.11\.0/2\.11\.1/g" opensearch-dashboards/transformVis/package.json && \ + sed -i "s/2\.11\.1/2\.12\.0/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \ + sed -i "s/2\.11\.1/2\.12\.0/g" opensearch-dashboards/transformVis/package.json && \ zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ cd /usr/share/opensearch-dashboards/plugins && \ /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/transformVis.zip --allow-root && \ diff --git a/Dockerfiles/opensearch.Dockerfile b/Dockerfiles/opensearch.Dockerfile index cb9aa6a66..19694ba80 100644 --- a/Dockerfiles/opensearch.Dockerfile +++ b/Dockerfiles/opensearch.Dockerfile @@ -1,4 +1,4 @@ -FROM opensearchproject/opensearch:2.11.1 +FROM opensearchproject/opensearch:2.12.0 # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" From b0756f2b4fc01b88e86d5f40925294cf6b37f96d Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 21 Feb 2024 07:41:44 -0700 Subject: [PATCH 13/79] now that dashboards custom branding allows relative urls, remove hacky overwrites --- Dockerfiles/dashboards.Dockerfile | 31 ---------------------------- Dockerfiles/nginx.Dockerfile | 9 ++++++-- dashboards/opensearch_dashboards.yml | 25 +++++++++------------- 3 files changed, 17 insertions(+), 48 deletions(-) diff --git a/Dockerfiles/dashboards.Dockerfile b/Dockerfiles/dashboards.Dockerfile index de1679bf4..b2ee0e30a 100644 --- a/Dockerfiles/dashboards.Dockerfile +++ b/Dockerfiles/dashboards.Dockerfile @@ -59,37 +59,6 @@ ADD dashboards/opensearch_dashboards.yml /usr/share/opensearch-dashboards/config ADD dashboards/scripts/docker_entrypoint.sh /usr/local/bin/ ADD scripts/malcolm_utils.py /usr/local/bin/ -# Yeah, I know about https://opensearch.org/docs/latest/dashboards/branding ... but I can't figure out a way -# to specify the entries in the opensearch_dashboards.yml such that they are valid BOTH from the -# internal opensearch code validating them AND the web browser retrieving them. So we're going scorched earth instead. - -COPY --chmod=644 docs/images/favicon/favicon192.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/android-chrome-192x192.png -COPY --chmod=644 docs/images/favicon/favicon512.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/android-chrome-512x512.png -COPY --chmod=644 docs/images/favicon/apple-touch-icon-precomposed.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/apple-touch-icon.png -COPY --chmod=644 docs/images/favicon/favicon16.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/favicon-16x16.png -COPY --chmod=644 docs/images/favicon/favicon32.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/favicon-32x32.png -COPY --chmod=644 docs/images/favicon/favicon.ico /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/favicon.ico -COPY --chmod=644 docs/images/favicon/favicon144.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/mstile-144x144.png -COPY --chmod=644 docs/images/favicon/favicon150.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/mstile-150x150.png -COPY --chmod=644 docs/images/favicon/favicon310.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/mstile-310x310.png -COPY --chmod=644 docs/images/favicon/favicon70.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/mstile-70x70.png -COPY --chmod=644 docs/images/logo/Malcolm.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch.svg -COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_center_mark.svg -COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_center_mark_on_dark.svg -COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_center_mark_on_light.svg -COPY --chmod=644 docs/images/logo/Malcolm.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_dashboards.svg -COPY --chmod=644 docs/images/logo/malcolm_logo.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_dashboards_on_dark.svg -COPY --chmod=644 docs/images/logo/Malcolm.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_dashboards_on_light.svg -COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_mark.svg -COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_mark_on_dark.svg -COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_mark_on_light.svg -COPY --chmod=644 docs/images/logo/malcolm_logo.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_on_dark.svg -COPY --chmod=644 docs/images/logo/Malcolm.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_on_light.svg -COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_spinner.svg -COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_spinner_on_dark.svg -COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_spinner_on_light.svg - - ENTRYPOINT ["/usr/bin/tini", \ "--", \ "/usr/local/bin/docker-uid-gid-setup.sh", \ diff --git a/Dockerfiles/nginx.Dockerfile b/Dockerfiles/nginx.Dockerfile index 5b3bdfd2d..ecb840a21 100644 --- a/Dockerfiles/nginx.Dockerfile +++ b/Dockerfiles/nginx.Dockerfile @@ -232,7 +232,9 @@ RUN set -x ; \ rm -rf /usr/src/* /var/tmp/* /var/cache/apk/* /nginx.tar.gz /nginx-auth-ldap.tar.gz /ngx_http_substitutions_filter_module-master.tar.gz; \ touch /etc/nginx/nginx_ldap.conf /etc/nginx/nginx_blank.conf && \ find /usr/share/nginx/html/ -type d -exec chmod 755 "{}" \; && \ - find /usr/share/nginx/html/ -type f -exec chmod 644 "{}" \; + find /usr/share/nginx/html/ -type f -exec chmod 644 "{}" \; && \ + cd /usr/share/nginx/html/assets/img && \ + ln -s ./Malcolm_background.png ./bg-masthead.png COPY --from=docbuild /site/_site /usr/share/nginx/html/readme @@ -244,7 +246,10 @@ ADD nginx/templates /etc/nginx/templates/ ADD nginx/supervisord.conf /etc/ COPY --chmod=644 docs/images/icon/favicon.ico /usr/share/nginx/html/assets/favicon.ico COPY --chmod=644 docs/images/icon/favicon.ico /usr/share/nginx/html/favicon.ico -COPY --chmod=644 docs/images/logo/Malcolm_background.png /usr/share/nginx/html/assets/img/bg-masthead.png +COPY --chmod=644 docs/images/icon/*.svg /usr/share/nginx/html/assets/img/ +COPY --chmod=644 docs/images/icon/*.png /usr/share/nginx/html/assets/img/ +COPY --chmod=644 docs/images/logo/*.svg /usr/share/nginx/html/assets/img/ +COPY --chmod=644 docs/images/logo/*.png /usr/share/nginx/html/assets/img/ VOLUME ["/etc/nginx/certs", "/etc/nginx/dhparam"] diff --git a/dashboards/opensearch_dashboards.yml b/dashboards/opensearch_dashboards.yml index 67d9f4f65..884c39533 100644 --- a/dashboards/opensearch_dashboards.yml +++ b/dashboards/opensearch_dashboards.yml @@ -17,21 +17,16 @@ data_source.enabled: false opensearchDashboards.branding: applicationTitle: "Malcolm Dashboards" useExpandedHeader: false - # Yeah, I know about https://opensearch.org/docs/latest/dashboards/branding ... but I can't figure out a way - # to specify the entries in the opensearch_dashboards.yml such that they are valid BOTH from the - # internal opensearch code validating them AND the web browser retrieving them. So we're going scorched earth instead - # by just overwriting the originals in our Dockerconfig. - # - # logo: - # defaultUrl: "http://dashboards:5601/dashboards/ui/assets/Malcolm.svg" - # darkModeUrl: "http://dashboards:5601/dashboards/ui/assets/malcolm_logo.svg" - # mark: - # defaultUrl: "http://dashboards:5601/dashboards/ui/assets/icon.png" - # darkModeUrl: "http://dashboards:5601/dashboards/ui/assets/icon_dark.png" - # loadingLogo: - # defaultUrl: "http://dashboards:5601/dashboards/ui/assets/icon.png" - # darkModeUrl: "http://dashboards:5601/dashboards/ui/assets/icon_dark.png" - # faviconUrl: "http://dashboards:5601/dashboards/ui/assets/favicon.ico" + logo: + defaultUrl: "/assets/img/Malcolm.svg" + darkModeUrl: "/assets/img/malcolm_logo.svg" + mark: + defaultUrl: "/assets/img/icon.png" + darkModeUrl: "/assets/img/icon_dark.png" + loadingLogo: + defaultUrl: "/assets/img/icon.png" + darkModeUrl: "/assets/img/icon_dark.png" + faviconUrl: "/assets/favicon.ico" map.regionmap: includeOpenSearchMapsService: false From 0879cea86e49fdb910680259ea2d08ddc00e3f7c Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 21 Feb 2024 07:44:04 -0700 Subject: [PATCH 14/79] for supporting JSON logs from Zeek (idaholab/Malcolm#65); almost certainly broken at this point --- logstash/pipelines/zeek/12_zeek_mutate.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash/pipelines/zeek/12_zeek_mutate.conf b/logstash/pipelines/zeek/12_zeek_mutate.conf index 321246729..f96ad0d4a 100644 --- a/logstash/pipelines/zeek/12_zeek_mutate.conf +++ b/logstash/pipelines/zeek/12_zeek_mutate.conf @@ -755,7 +755,7 @@ filter { query = uri.nil? ? nil : uri.partition('?').last event.set('[url][scheme]', scheme) event.set('[url][original]', scheme + '://' + (host.nil? ? '' : host) + (uri.nil? ? '' : uri)) - event.set('[url][full]', scheme + '://' + (user.nil? ? '' : Array(user).first) + (password.nil? ? '' : ':' + password) + ((user.nil? && password.nil?) ? '' : '@') + (host.nil? ? '' : host) + (port.nil? ? '' : ':' + port) + (uri.nil? ? '' : uri)) + event.set('[url][full]', scheme + '://' + (user.nil? ? '' : Array(user).first) + (password.nil? ? '' : ':' + password) + ((user.nil? && password.nil?) ? '' : '@') + (host.nil? ? '' : host) + (port.nil? ? '' : ':' + port.to_s) + (uri.nil? ? '' uri)) event.set('[url][domain]', host) unless host.nil? event.set('[url][extension]', ext) unless ext.nil? || ext.empty? event.set('[url][fragment]', fragment) unless fragment.nil? || fragment.empty? From 26f723867023bc96ee8cda89f6192d278b943c70 Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 21 Feb 2024 07:48:46 -0700 Subject: [PATCH 15/79] for supporting JSON logs from Zeek (idaholab/Malcolm#65); almost certainly broken at this point --- logstash/pipelines/zeek/12_zeek_mutate.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash/pipelines/zeek/12_zeek_mutate.conf b/logstash/pipelines/zeek/12_zeek_mutate.conf index f96ad0d4a..85a026b67 100644 --- a/logstash/pipelines/zeek/12_zeek_mutate.conf +++ b/logstash/pipelines/zeek/12_zeek_mutate.conf @@ -755,7 +755,7 @@ filter { query = uri.nil? ? nil : uri.partition('?').last event.set('[url][scheme]', scheme) event.set('[url][original]', scheme + '://' + (host.nil? ? '' : host) + (uri.nil? ? '' : uri)) - event.set('[url][full]', scheme + '://' + (user.nil? ? '' : Array(user).first) + (password.nil? ? '' : ':' + password) + ((user.nil? && password.nil?) ? '' : '@') + (host.nil? ? '' : host) + (port.nil? ? '' : ':' + port.to_s) + (uri.nil? ? '' uri)) + event.set('[url][full]', scheme + '://' + (user.nil? ? '' : Array(user).first) + (password.nil? ? '' : ':' + password) + ((user.nil? && password.nil?) ? '' : '@') + (host.nil? ? '' : host) + (port.nil? ? '' : ':' + port.to_s) + (uri.nil? ? '' : uri)) event.set('[url][domain]', host) unless host.nil? event.set('[url][extension]', ext) unless ext.nil? || ext.empty? event.set('[url][fragment]', fragment) unless fragment.nil? || fragment.empty? From 096e32c42829fb2a75982939feb0db1eb98dea24 Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 21 Feb 2024 08:30:27 -0700 Subject: [PATCH 16/79] for supporting JSON logs from Zeek (idaholab/Malcolm#65); getting closer --- logstash/pipelines/zeek/11_zeek_parse.conf | 20 ++++++++++++------- .../pipelines/zeek/13_zeek_normalize.conf | 4 ++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index 8ea1bbff5..5b945d390 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -2206,18 +2206,18 @@ filter { split => { "[zeek_cols][actions]" => "," } } } - if (![zeek_cols][orig_h]) or ([zeek_cols][orig_h] == '(empty)') or - ([zeek_cols][orig_h] == '-') or ([zeek_cols][orig_h] == '') { + if ([zeek_cols][src]) and ((![zeek_cols][orig_h]) or ([zeek_cols][orig_h] == '(empty)') or + ([zeek_cols][orig_h] == '-') or ([zeek_cols][orig_h] == '')) { mutate { id => "mutate_replace_zeek_notice_orig_h" replace => { "[zeek_cols][orig_h]" => "%{[zeek_cols][src]}" } } } - if (![zeek_cols][resp_h]) or ([zeek_cols][resp_h] == '(empty)') or - ([zeek_cols][resp_h] == '-') or ([zeek_cols][resp_h] == '') { + if ([zeek_cols][dst]) and ((![zeek_cols][resp_h]) or ([zeek_cols][resp_h] == '(empty)') or + ([zeek_cols][resp_h] == '-') or ([zeek_cols][resp_h] == '')) { mutate { id => "mutate_replace_zeek_notice_resp_h" replace => { "[zeek_cols][resp_h]" => "%{[zeek_cols][dst]}" } } } - if (![zeek_cols][resp_p]) or ([zeek_cols][resp_p] == '(empty)') or - ([zeek_cols][resp_p] == '-') or ([zeek_cols][resp_p] == '') { + if [zeek_cols][p] and ((![zeek_cols][resp_p]) or ([zeek_cols][resp_p] == '(empty)') or + ([zeek_cols][resp_p] == '-') or ([zeek_cols][resp_p] == '')) { mutate { id => "mutate_replace_zeek_resp_p" replace => { "[zeek_cols][resp_p]" => "%{[zeek_cols][p]}" } } } @@ -3530,7 +3530,13 @@ filter { # tds_rpc.log # https://github.com/amzn/zeek-plugin-tds/blob/master/scripts/main.zeek - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_tds_rpc_fields" + rename => { "[zeek_cols][parameters]" => "[zeek_cols][parameter]" } + } + + } else { dissect { id => "dissect_zeek_tds_rpc" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP diff --git a/logstash/pipelines/zeek/13_zeek_normalize.conf b/logstash/pipelines/zeek/13_zeek_normalize.conf index 48d97d88d..a57d65cb3 100644 --- a/logstash/pipelines/zeek/13_zeek_normalize.conf +++ b/logstash/pipelines/zeek/13_zeek_normalize.conf @@ -81,8 +81,8 @@ filter { id => "ruby_zeek_field_network_protocol_version_ipsec" code => " versions = Array.new - versions << [event.get('[zeek][ipsec][maj_ver]'), - event.get('[zeek][ipsec][min_ver]')].reject{ |e| e.nil? || e&.empty? }.join('.') + versions << [event.get('[zeek][ipsec][maj_ver]').to_s, + event.get('[zeek][ipsec][min_ver]').to_s].reject{ |e| e.nil? || e&.empty? }.join('.') event.set('[network][protocol_version]', versions)" } } From 57e0141a34f2f4274d4fe7247e338d03adcde48c Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 21 Feb 2024 09:14:00 -0700 Subject: [PATCH 17/79] update osd vis plugin --- Dockerfiles/dashboards.Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfiles/dashboards.Dockerfile b/Dockerfiles/dashboards.Dockerfile index b2ee0e30a..3285435cd 100644 --- a/Dockerfiles/dashboards.Dockerfile +++ b/Dockerfiles/dashboards.Dockerfile @@ -20,7 +20,7 @@ ENV PUSER_PRIV_DROP true ENV TERM xterm ENV TINI_VERSION v0.19.0 -ENV OSD_TRANSFORM_VIS_VERSION 2.11.1 +ENV OSD_TRANSFORM_VIS_VERSION 2.12.0 ARG NODE_OPTIONS="--max_old_space_size=4096" ENV NODE_OPTIONS $NODE_OPTIONS @@ -39,10 +39,10 @@ RUN yum upgrade -y && \ # Malcolm manages authentication and encryption via NGINX reverse proxy /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityDashboards --allow-root && \ cd /tmp && \ - unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ - sed -i "s/2\.11\.1/2\.12\.0/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \ - sed -i "s/2\.11\.1/2\.12\.0/g" opensearch-dashboards/transformVis/package.json && \ - zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ + # unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ + # sed -i "s/2\.12\.0/2\.12\.0/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \ + # sed -i "s/2\.12\.0/2\.12\.0/g" opensearch-dashboards/transformVis/package.json && \ + # zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ cd /usr/share/opensearch-dashboards/plugins && \ /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/transformVis.zip --allow-root && \ rm -rf /tmp/transformVis /tmp/opensearch-dashboards && \ From c2826f8ae2ff6d273442aded41950906acc627c2 Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 21 Feb 2024 09:29:11 -0700 Subject: [PATCH 18/79] for supporting JSON logs from Zeek (idaholab/Malcolm#65); getting closer --- logstash/pipelines/zeek/11_zeek_parse.conf | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index 5b945d390..5466db53c 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -28,7 +28,21 @@ filter { # in JSON, do some global renaming of common fields to make them match the names we'd # be assigning to them if we were reading TSV if ("_jsonparsesuccess" in [tags]) { - if ("" in [zeek_cols][is_orig]) and (([zeek_cols][source_h]) or ([zeek_cols][destination_h])) { + + ruby { + id => "ruby_zeek_json_determine_source_destination_fields" + code => " + if !event.get('[zeek_cols][is_orig]').nil? and + ![event.get('[zeek_cols][source_h]').to_s, + event.get('[zeek_cols][source_p]').to_s, + event.get('[zeek_cols][destination_h]').to_s, + event.get('[zeek_cols][destination_p]').to_s].reject{ |e| e.nil? || e.empty? || (e == '0') }.empty? then + event.set('[@metadata][icsnpp_source_dest_fields]', 'true') + end + " + } + + if ([@metadata][icsnpp_source_dest_fields]) { # some of the ICSNPP parsers do an interesting thing to handle source and destination fields # (see https://github.com/cisagov/icsnpp-bacnet/?tab=readme-ov-file#source-and-destination-fields) # so check for and handle those first @@ -43,6 +57,7 @@ filter { rename => { "[zeek_cols][destination_h]" => "[zeek_cols][resp_h]" } rename => { "[zeek_cols][destination_p]" => "[zeek_cols][resp_p]" } } + } else { mutate { id => "mutate_rename_zeek_json_common_fields" @@ -51,8 +66,8 @@ filter { rename => { "[zeek_cols][id.resp_h]" => "[zeek_cols][resp_h]" } rename => { "[zeek_cols][id.resp_p]" => "[zeek_cols][resp_p]" } } - } - } + } # icsnpp_source_dest_fields or not + } # _jsonparsesuccess in tags # The Dissect is WAY faster than CSV, and quite a bit faster than mutate.split. However, it # is not as flexible when it comes to missing or extra columns From 540c0ebcffa74d1150c8bd9bf3b3e7f706d97456 Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 21 Feb 2024 10:51:40 -0700 Subject: [PATCH 19/79] for supporting JSON logs from Zeek (idaholab/Malcolm#65); getting closer --- logstash/pipelines/zeek/11_zeek_parse.conf | 143 ++++++++++++++++++--- 1 file changed, 128 insertions(+), 15 deletions(-) diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index 5466db53c..02ef56037 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -659,6 +659,7 @@ filter { if ("_jsonparsesuccess" in [tags]) { mutate { id => "mutate_rename_zeek_json_dhcp_fields" + rename => { "[zeek_cols][uids]" => "[zeek_cols][uid]" } rename => { "[zeek_cols][client_addr]" => "[zeek_cols][orig_h]" } rename => { "[zeek_cols][server_addr]" => "[zeek_cols][orig_p]" } rename => { "[zeek_cols][mac]" => "[zeek_cols][orig_l2_addr]" } @@ -862,7 +863,13 @@ filter { # dpd.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/dpd/main.zeek.html#type-DPD::Info - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_dpd_fields" + rename => { "[zeek_cols][analyzer]" => "[zeek_cols][service]" } + } + + } else { dissect { id => "dissect_zeek_dpd" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -921,7 +928,7 @@ filter { } ruby { id => "ruby_zip_zeek_enip" - init => "@zeek_enip_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'enip_command', 'length', 'session_handle', 'enip_status', 'sender_context', 'options' ]" + init => "@zeek_enip_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'enip_command_code', 'enip_command', 'length', 'session_handle', 'enip_status', 'sender_context', 'options' ]" code => "event.set('[zeek_cols]', @zeek_enip_field_names.zip(event.get('[message]')).to_h)" } } @@ -938,7 +945,18 @@ filter { # ecat_registers.log # https://github.com/cisagov/icsnpp-ethercat - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_ecat_registers_fields" + rename => { "[zeek_cols][srcmac]" => "[zeek_cols][orig_l2_addr]" } + rename => { "[zeek_cols][dstmac]" => "[zeek_cols][resp_l2_addr]" } + rename => { "[zeek_cols][Command]" => "[zeek_cols][command]" } + rename => { "[zeek_cols][Slave_Addr]" => "[zeek_cols][server_addr]" } + rename => { "[zeek_cols][Register_Type]" => "[zeek_cols][register_type]" } + rename => { "[zeek_cols][Register_Addr]" => "[zeek_cols][register_addr]" } + } + + } else { dissect { id => "dissect_zeek_ecat_registers" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -973,7 +991,17 @@ filter { # ecat_log_address.log # https://github.com/cisagov/icsnpp-ethercat - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_ecat_log_address_fields" + rename => { "[zeek_cols][srcmac]" => "[zeek_cols][orig_l2_addr]" } + rename => { "[zeek_cols][dstmac]" => "[zeek_cols][resp_l2_addr]" } + rename => { "[zeek_cols][Log_Addr]" => "[zeek_cols][log_addr]" } + rename => { "[zeek_cols][Length]" => "[zeek_cols][length]" } + rename => { "[zeek_cols][Command]" => "[zeek_cols][command]" } + } + + } else { dissect { id => "dissect_zeek_ecat_log_address" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1008,7 +1036,13 @@ filter { # ecat_dev_info.log # https://github.com/cisagov/icsnpp-ethercat - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_ecat_dev_info_fields" + rename => { "[zeek_cols][slave_id]" => "[zeek_cols][server_id]" } + } + + } else { dissect { id => "dissect_zeek_ecat_dev_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1043,7 +1077,18 @@ filter { # ecat_aoe_info.log # https://github.com/cisagov/icsnpp-ethercat - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_ecat_aoe_info_fields" + rename => { "[zeek_cols][targetid]" => "[zeek_cols][resp_l2_addr]" } + rename => { "[zeek_cols][targetport]" => "[zeek_cols][resp_port]" } + rename => { "[zeek_cols][senderid]" => "[zeek_cols][orig_l2_addr]" } + rename => { "[zeek_cols][senderport]" => "[zeek_cols][orig_port]" } + rename => { "[zeek_cols][cmd]" => "[zeek_cols][command]" } + rename => { "[zeek_cols][stateflags]" => "[zeek_cols][state]" } + } + + } else { dissect { id => "dissect_zeek_ecat_aoe_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1078,7 +1123,13 @@ filter { # ecat_coe_info.log # https://github.com/cisagov/icsnpp-ethercat - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_ecat_coe_info_fields" + rename => { "[zeek_cols][Type]" => "[zeek_cols][type]" } + } + + } else { dissect { id => "dissect_zeek_ecat_coe_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1113,7 +1164,13 @@ filter { # ecat_foe_info.log # https://github.com/cisagov/icsnpp-ethercat - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_ecat_foe_info_fields" + rename => { "[zeek_cols][opCode]" => "[zeek_cols][opcode]" } + } + + } else { dissect { id => "dissect_zeek_ecat_foe_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1148,7 +1205,14 @@ filter { # ecat_soe_info.log # https://github.com/cisagov/icsnpp-ethercat - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_ecat_soe_info_fields" + rename => { "[zeek_cols][opCode]" => "[zeek_cols][opcode]" } + rename => { "[zeek_cols][element_flags]" => "[zeek_cols][element]" } + } + + } else { dissect { id => "dissect_zeek_ecat_soe_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1187,7 +1251,18 @@ filter { # to be handled bdrop_zeek_ignored_source above, as it is including ALL arps and not just those # from ethercat traffic which can be misleading (i.e., indicating ecat traffic where there is none) - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_ecat_arp_info_fields" + rename => { "[zeek_cols][mac_src]" => "[zeek_cols][orig_l2_addr]" } + rename => { "[zeek_cols][mac_dst]" => "[zeek_cols][resp_l2_addr]" } + rename => { "[zeek_cols][SPA]" => "[zeek_cols][orig_proto_addr]" } + rename => { "[zeek_cols][SHA]" => "[zeek_cols][orig_hw_addr]" } + rename => { "[zeek_cols][TPA]" => "[zeek_cols][resp_proto_addr]" } + rename => { "[zeek_cols][THA]" => "[zeek_cols][resp_hw_addr]" } + } + + } else { dissect { id => "dissect_zeek_ecat_arp_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1324,7 +1399,13 @@ filter { # genisys.log # https://github.com/cisagov/icsnpp-genisys - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_genisys_fields" + rename => { "[zeek_cols][payload]" => "[zeek_cols][payload_raw]" } + } + + } else { dissect { id => "dissect_zeek_genisys" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1909,7 +1990,15 @@ filter { # modbus_detailed.log # main.zeek (https://github.com/cisagov/icsnpp-modbus) - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_modbus_detailed_fields" + rename => { "[zeek_cols][tid]" => "[zeek_cols][trans_id]" } + rename => { "[zeek_cols][unit]" => "[zeek_cols][unit_id]" } + rename => { "[zeek_cols][request_response]" => "[zeek_cols][network_direction]" } + } + + } else { dissect { id => "dissect_zeek_modbus_detailed" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1946,7 +2035,15 @@ filter { # modbus_mask_write_register.log # main.zeek (https://github.com/cisagov/icsnpp-modbus) - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_modbus_mask_write_register_fields" + rename => { "[zeek_cols][tid]" => "[zeek_cols][trans_id]" } + rename => { "[zeek_cols][unit]" => "[zeek_cols][unit_id]" } + rename => { "[zeek_cols][request_response]" => "[zeek_cols][network_direction]" } + } + + } else { dissect { id => "dissect_zeek_modbus_mask_write_register" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1981,7 +2078,15 @@ filter { # modbus_read_device_identification.log # main.zeek (https://github.com/cisagov/icsnpp-modbus) - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_modbus_read_device_identification_fields" + rename => { "[zeek_cols][tid]" => "[zeek_cols][trans_id]" } + rename => { "[zeek_cols][unit]" => "[zeek_cols][unit_id]" } + rename => { "[zeek_cols][request_response]" => "[zeek_cols][network_direction]" } + } + + } else { dissect { id => "dissect_zeek_modbus_read_device_identification" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2016,7 +2121,15 @@ filter { # modbus_read_write_multiple_registers.log # main.zeek (https://github.com/cisagov/icsnpp-modbus) - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_modbus_read_write_multiple_registers_fields" + rename => { "[zeek_cols][tid]" => "[zeek_cols][trans_id]" } + rename => { "[zeek_cols][unit]" => "[zeek_cols][unit_id]" } + rename => { "[zeek_cols][request_response]" => "[zeek_cols][network_direction]" } + } + + } else { dissect { id => "dissect_zeek_modbus_read_write_multiple_registers" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP From 5476493570ef24a829e49f64101367cdcd575890 Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 21 Feb 2024 14:13:48 -0700 Subject: [PATCH 20/79] for supporting JSON logs from Zeek (idaholab/Malcolm#65); getting closer --- logstash/pipelines/zeek/11_zeek_parse.conf | 204 ++++++++++++++++++--- 1 file changed, 182 insertions(+), 22 deletions(-) diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index 02ef56037..37e2d84c3 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -1475,7 +1475,13 @@ filter { # http.log # https://docs.zeek.org/en/stable/scripts/base/protocols/http/main.zeek.html#type-HTTP::Info - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_http_fields" + rename => { "[zeek_cols][username]" => "[zeek_cols][user]" } + } + + } else { dissect { id => "dissect_zeek_http_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1663,7 +1669,14 @@ filter { # kerberos.log # https://docs.zeek.org/en/stable/scripts/base/protocols/krb/main.zeek.html#type-KRB::Info - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_kerberos_fields" + rename => { "[zeek_cols][client]" => "[zeek_cols][cname]" } + rename => { "[zeek_cols][service]" => "[zeek_cols][sname]" } + } + + } else { dissect { id => "dissect_zeek_kerberos" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1698,7 +1711,14 @@ filter { # known_certs.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/ssl/known-certs.zeek.html#type-Known::CertsInfo - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_known_certs_fields" + rename => { "[zeek_cols][host]" => "[zeek_cols][orig_h]" } + rename => { "[zeek_cols][port_num]" => "[zeek_cols][orig_p]" } + } + + } else { dissect { id => "dissect_zeek_known_certs" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1760,7 +1780,13 @@ filter { # known_modbus.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/modbus/known-masters-slaves.zeek.html#type-Known::ModbusInfo - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_known_modbus_fields" + rename => { "[zeek_cols][host]" => "[zeek_cols][orig_h]" } + } + + } else { dissect { id => "dissect_zeek_known_modbus" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1799,7 +1825,15 @@ filter { # known_services.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/conn/known-services.zeek.html#type-Known::ServicesInfo - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_known_services_fields" + rename => { "[zeek_cols][host]" => "[zeek_cols][resp_h]" } + rename => { "[zeek_cols][port_num]" => "[zeek_cols][resp_p]" } + rename => { "[zeek_cols][port_proto]" => "[zeek_cols][proto]" } + } + + } else { dissect { id => "dissect_zeek_known_services" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1847,7 +1881,16 @@ filter { # ldap.log # main.zeek (https://docs.zeek.org/en/master/scripts/base/protocols/ldap/main.zeek.html) - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_ldap_fields" + rename => { "[zeek_cols][arguments]" => "[zeek_cols][argument]" } + rename => { "[zeek_cols][opcodes]" => "[zeek_cols][operation]" } + rename => { "[zeek_cols][results]" => "[zeek_cols][result_code]" } + rename => { "[zeek_cols][diagnostic_message]" => "[zeek_cols][result_message]" } + } + + } else { dissect { id => "dissect_zeek_ldap" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1888,7 +1931,18 @@ filter { # ldap_search.log # main.zeek (https://docs.zeek.org/en/master/scripts/base/protocols/ldap/main.zeek.html) - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_ldap_search_fields" + rename => { "[zeek_cols][deref_aliases]" => "[zeek_cols][deref]" } + rename => { "[zeek_cols][derefs]" => "[zeek_cols][deref]" } + rename => { "[zeek_cols][base_objects]" => "[zeek_cols][base_object]" } + rename => { "[zeek_cols][results]" => "[zeek_cols][result_code]" } + rename => { "[zeek_cols][result]" => "[zeek_cols][result_code]" } + rename => { "[zeek_cols][diagnostic_message]" => "[zeek_cols][result_message]" } + } + + } else { dissect { id => "dissect_zeek_ldap_search" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2355,7 +2409,18 @@ filter { # ntlm.log # https://docs.zeek.org/en/stable/scripts/base/protocols/ntlm/main.zeek.html#type-NTLM::Info - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_ntlm_fields" + rename => { "[zeek_cols][hostname]" => "[zeek_cols][host]" } + rename => { "[zeek_cols][domainname]" => "[zeek_cols][domain]" } + rename => { "[zeek_cols][server_nb_computer_name]" => "[zeek_cols][server_nb_computer]" } + rename => { "[zeek_cols][server_dns_computer_name]" => "[zeek_cols][server_dns_computer]" } + rename => { "[zeek_cols][server_tree_name]" => "[zeek_cols][server_tree]" } + rename => { "[zeek_cols][username]" => "[zeek_cols][user]" } + } + + } else { dissect { id => "dissect_zeek_ntlm_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2453,7 +2518,14 @@ filter { # ospf.log # https://github.com/corelight/zeek-spicy-ospf - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_ospf_fields" + rename => { "[zeek_cols][ip_dst]" => "[zeek_cols][orig_h]" } + rename => { "[zeek_cols][ip_src]" => "[zeek_cols][resp_h]" } + } + + } else { dissect { id => "dissect_zeek_ospf" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2497,7 +2569,13 @@ filter { # pe.log # https://docs.zeek.org/en/stable/scripts/base/files/pe/main.zeek.html#type-PE::Info - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_pe_fields" + rename => { "[zeek_cols][id]" => "[zeek_cols][fuid]" } + } + + } else { dissect { id => "dissect_zeek_pe" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2592,7 +2670,13 @@ filter { # radius.log # https://docs.zeek.org/en/stable/scripts/base/protocols/radius/main.zeek.html#type-RADIUS::Info - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_radius_fields" + rename => { "[zeek_cols][username]" => "[zeek_cols][user]" } + } + + } else { dissect { id => "dissect_zeek_radius" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2802,7 +2886,13 @@ filter { # s7comm_upload_download.log # https://github.com/cisagov/icsnpp-s7comm - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_s7comm_upload_download_fields" + rename => { "[zeek_cols][rosctr]" => "[zeek_cols][rosctr_name]" } + } + + } else { dissect { id => "dissect_zeek_s7comm_upload_download" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -2908,7 +2998,16 @@ filter { # note that smb_cmd.referenced_file is exactly the same structure as the log line for smb_files. later on it will be # merged up as its own top-level entity so I don't have to duplicate the parsing effort below - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_smb_cmd_referenced_file_fields" + rename => { "[zeek_cols][referenced_file.id.orig_h]" => "[zeek_cols][referenced_file.orig_h]" } + rename => { "[zeek_cols][referenced_file.id.orig_p]" => "[zeek_cols][referenced_file.orig_p]" } + rename => { "[zeek_cols][referenced_file.id.resp_h]" => "[zeek_cols][referenced_file.resp_h]" } + rename => { "[zeek_cols][referenced_file.id.resp_p]" => "[zeek_cols][referenced_file.resp_p]" } + } + + } else { dissect { id => "dissect_zeek_smb_cmd" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3130,7 +3229,19 @@ filter { # software.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/software/main.zeek.html#type-Software::Info - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_software_fields" + rename => { "[zeek_cols][host]" => "[zeek_cols][orig_h]" } + rename => { "[zeek_cols][host_p]" => "[zeek_cols][orig_p]" } + rename => { "[zeek_cols][version.major]" => "[zeek_cols][version_major]" } + rename => { "[zeek_cols][version.minor]" => "[zeek_cols][version_minor]" } + rename => { "[zeek_cols][version.minor2]" => "[zeek_cols][version_minor2]" } + rename => { "[zeek_cols][version.minor3]" => "[zeek_cols][version_minor3]" } + rename => { "[zeek_cols][version.addl]" => "[zeek_cols][version_addl]" } + } + + } else { dissect { id => "dissect_zeek_software" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3235,7 +3346,13 @@ filter { # ssl.log # https://docs.zeek.org/en/stable/scripts/base/protocols/ssl/main.zeek.html#type-SSL::Info - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_ssl_fields" + rename => { "[zeek_cols][version]" => "[zeek_cols][ssl_version]" } + } + + } else { dissect { id => "dissect_zeek_ssl_v1_with_ja3" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3275,7 +3392,14 @@ filter { # stun.log # https://github.com/corelight/zeek-spicy-stun/blob/master/analyzer/main.zeek - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_stun_fields" + rename => { "[zeek_cols][attr_types]" => "[zeek_cols][attr_type]" } + rename => { "[zeek_cols][attr_vals]" => "[zeek_cols][attr_val]" } + } + + } else { dissect { id => "dissect_zeek_stun" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3295,8 +3419,9 @@ filter { code => "event.set('[zeek_cols]', @zeek_stun_field_names.zip(event.get('[message]')).to_h)" } } - mutate { id => "mutate_split_field_zeek_stun_attr_type" - split => { "[zeek_cols][attr_type]" => "," } } + mutate { id => "mutate_split_field_zeek_stun_commas" + split => { "[zeek_cols][attr_type]" => "," + "[zeek_cols][attr_val]" => "," } } } mutate { @@ -3311,7 +3436,15 @@ filter { # stun.log # https://github.com/corelight/zeek-spicy-stun/blob/master/analyzer/main.zeek - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_stun_nat_fields" + rename => { "[zeek_cols][wan_addrs]" => "[zeek_cols][wan_addr]" } + rename => { "[zeek_cols][wan_ports]" => "[zeek_cols][wan_port]" } + rename => { "[zeek_cols][lan_addrs]" => "[zeek_cols][lan_addr]" } + } + + } else { dissect { id => "dissect_zeek_stun_nat" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -3331,6 +3464,10 @@ filter { code => "event.set('[zeek_cols]', @zeek_stun_nat_field_names.zip(event.get('[message]')).to_h)" } } + mutate { id => "mutate_split_field_zeek_stun_nat_commas" + split => { "[zeek_cols][wan_addrs]" => "," + "[zeek_cols][wan_ports]" => "," + "[zeek_cols][lan_addrs]" => "," } } } mutate { @@ -3815,7 +3952,7 @@ filter { id => "dissect_zeek_weird" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][name]} %{[zeek_cols][addl]} %{[zeek_cols][notice]} %{[zeek_cols][peer]}" + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][name]} %{[zeek_cols][addl]} %{[zeek_cols][notice]} %{[zeek_cols][peer]} %{[zeek_cols][source]}" } } if ("_dissectfailure" in [tags]) { @@ -3826,7 +3963,7 @@ filter { } ruby { id => "ruby_zip_zeek_weird" - init => "@zeek_weird_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'name', 'addl', 'notice', 'peer' ]" + init => "@zeek_weird_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'name', 'addl', 'notice', 'peer', 'source' ]" code => "event.set('[zeek_cols]', @zeek_weird_field_names.zip(event.get('[message]')).to_h)" } } @@ -3837,7 +3974,30 @@ filter { # x509.log # https://docs.zeek.org/en/stable/scripts/base/files/x509/main.zeek.html#type-X509::Info - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_x509_fields" + rename => { "[zeek_cols][certificate.version]" => "[zeek_cols][certificate_version]" } + rename => { "[zeek_cols][certificate.serial]" => "[zeek_cols][certificate_serial]" } + rename => { "[zeek_cols][certificate.subject]" => "[zeek_cols][certificate_subject]" } + rename => { "[zeek_cols][certificate.issuer]" => "[zeek_cols][certificate_issuer]" } + rename => { "[zeek_cols][certificate.not_valid_before]" => "[zeek_cols][certificate_not_valid_before]" } + rename => { "[zeek_cols][certificate.not_valid_after]" => "[zeek_cols][certificate_not_valid_after]" } + rename => { "[zeek_cols][certificate.key_alg]" => "[zeek_cols][certificate_key_alg]" } + rename => { "[zeek_cols][certificate.sig_alg]" => "[zeek_cols][certificate_sig_alg]" } + rename => { "[zeek_cols][certificate.key_type]" => "[zeek_cols][certificate_key_type]" } + rename => { "[zeek_cols][certificate.key_length]" => "[zeek_cols][certificate_key_length]" } + rename => { "[zeek_cols][certificate.exponent]" => "[zeek_cols][certificate_exponent]" } + rename => { "[zeek_cols][certificate.curve]" => "[zeek_cols][certificate_curve]" } + rename => { "[zeek_cols][san.dns]" => "[zeek_cols][san_dns]" } + rename => { "[zeek_cols][san.uri]" => "[zeek_cols][san_uri]" } + rename => { "[zeek_cols][san.email]" => "[zeek_cols][san_email]" } + rename => { "[zeek_cols][san.ip]" => "[zeek_cols][san_ip]" } + rename => { "[zeek_cols][basic_constraints.ca]" => "[zeek_cols][basic_constraints_ca]" } + rename => { "[zeek_cols][basic_constraints.path_len]" => "[zeek_cols][basic_constraints_path_len]" } + } + + } else { dissect { id => "dissect_zeek_x509_v1" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP From d9021c992b6fcd800f7f5b9637e2bdb2c2142caa Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 21 Feb 2024 15:00:26 -0700 Subject: [PATCH 21/79] for supporting JSON logs from Zeek (idaholab/Malcolm#65); getting closer --- arkime/etc/config.ini | 8 ++++---- arkime/wise/source.zeeklogs.js | 1 + .../templates/composable/component/zeek.json | 2 ++ logstash/pipelines/zeek/11_zeek_parse.conf | 14 +++++++------- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/arkime/etc/config.ini b/arkime/etc/config.ini index a34b194b3..1dedadba3 100644 --- a/arkime/etc/config.ini +++ b/arkime/etc/config.ini @@ -1694,6 +1694,7 @@ zeek.software.version_minor2=db:zeek.software.version_minor2;group:zeek_software zeek.software.version_minor3=db:zeek.software.version_minor3;group:zeek_software;kind:integer;viewerOnly:true;friendly:Minor Patch;help:Minor Patch zeek.software.version_addl=db:zeek.software.version_addl;group:zeek_software;kind:termfield;viewerOnly:true;friendly:Additional Version;help:Additional Version zeek.software.unparsed_version=db:zeek.software.unparsed_version;group:zeek_software;kind:termfield;viewerOnly:true;friendly:Version;help:Version +zeek.software.url=db:zeek.software.url;group:zeek_software;kind:termfield;viewerOnly:true;friendly:URL;help:URL # ssh.log # https://docs.zeek.org/en/stable/scripts/base/protocols/ssh/main.zeek.html#type-SSH::Info @@ -1963,10 +1964,9 @@ zeek.tunnel.action=db:zeek.tunnel.action;group:zeek_tunnel;kind:termfield;viewer # weird.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/notice/weird.zeek.html#type-Weird::Info -zeek.weird.name=db:zeek.weird.name;group:zeek_weird;kind:termfield;viewerOnly:true;friendly:Name;help:Name zeek.weird.addl=db:zeek.weird.addl;group:zeek_weird;kind:termfield;viewerOnly:true;friendly:Additional Info;help:Additional Info zeek.weird.notice=db:zeek.weird.notice;group:zeek_weird;kind:termfield;viewerOnly:true;friendly:Notice;help:Generated a notice -zeek.weird.peer=db:zeek.weird.peer;group:zeek_weird;kind:termfield;viewerOnly:true;friendly:Remote Peer;help:Remote Peer +zeek.weird.source=db:zeek.weird.source;group:zeek_weird;kind:termfield;viewerOnly:true;friendly:Weird Source;help:Weird Source # wireguard.log # https://github.com/corelight/zeek-spicy-wireguard/blob/master/analyzer/main.zeek @@ -2663,7 +2663,7 @@ o_zeek_smb_mapping=require:zeek.smb_mapping;title:Zeek smb_mapping.log;fields:ze o_zeek_smtp=require:zeek.smtp;title:Zeek smtp.log;fields:zeek.smtp.trans_depth,zeek.smtp.helo,zeek.smtp.mailfrom,zeek.smtp.rcptto,zeek.smtp.date,zeek.smtp.from,zeek.smtp.to,zeek.smtp.cc,zeek.smtp.reply_to,zeek.smtp.msg_id,zeek.smtp.in_reply_to,zeek.smtp.subject,zeek.smtp.x_originating_ip,zeek.smtp.first_received,zeek.smtp.second_received,zeek.smtp.last_reply,zeek.smtp.last_reply_code,zeek.smtp.last_reply_msg,zeek.smtp.path,zeek.smtp.user_agent,zeek.smtp.tls,zeek.smtp.is_webmail o_zeek_snmp=require:zeek.snmp;title:Zeek snmp.log;fields:zeek.snmp.duration,zeek.snmp.version,zeek.snmp.community,zeek.snmp.get_requests,zeek.snmp.get_bulk_requests,zeek.snmp.get_responses,zeek.snmp.set_requests,zeek.snmp.display_string,zeek.snmp.up_since o_zeek_socks=require:zeek.socks;title:Zeek socks.log;fields:zeek.socks.version,zeek.socks.server_status,zeek.socks.request_host,zeek.socks.request_name,zeek.socks.request_port,zeek.socks.bound_host,zeek.socks.bound_name,zeek.socks.bound_port -o_zeek_software=require:zeek.software;title:Zeek software.log;fields:zeek.software.software_type,zeek.software.name,zeek.software.version_major,zeek.software.version_minor,zeek.software.version_minor2,zeek.software.version_minor3,zeek.software.version_addl,zeek.software.unparsed_version +o_zeek_software=require:zeek.software;title:Zeek software.log;fields:zeek.software.software_type,zeek.software.name,zeek.software.version_major,zeek.software.version_minor,zeek.software.version_minor2,zeek.software.version_minor3,zeek.software.version_addl,zeek.software.unparsed_version,zeek.software.url o_zeek_ssh=require:zeek.ssh;title:Zeek ssh.log;fields:zeek.ssh.version,zeek.ssh.auth_success,zeek.ssh.auth_attempts,zeek.ssh.direction,zeek.ssh.client,zeek.ssh.server,zeek.ssh.cipher_alg,zeek.ssh.mac_alg,zeek.ssh.compression_alg,zeek.ssh.kex_alg,zeek.ssh.host_key_alg,zeek.ssh.host_key,zeek.ssh.remote_location_country_code,zeek.ssh.remote_location_region,zeek.ssh.remote_location_city,zeek.ssh.remote_location_latitude,zeek.ssh.remote_location_longitude,zeek.ssh.hassh,zeek.ssh.hasshServer,zeek.ssh.hasshAlgorithms,zeek.ssh.hasshServerAlgorithms,zeek.ssh.cshka,zeek.ssh.sshka o_zeek_ssl=require:zeek.ssl;title:Zeek ssl.log;fields:zeek.ssl.ssl_version,zeek.ssl.ssl_history,zeek.ssl.sni_matches_cert,zeek.ssl.cipher,zeek.ssl.curve,zeek.ssl.server_name,zeek.ssl.resumed,zeek.ssl.last_alert,zeek.ssl.next_protocol,zeek.ssl.established,tls.client.ja3,tls.client.ja3_description,tls.server.ja3s,tls.server.ja3s_description,zeek.ssl.cert_chain_fuids,zeek.ssl.cert_chain_fps,zeek.ssl.client_cert_chain_fuids,zeek.ssl.client_cert_chain_fps,zeek.ssl.subject.CN,zeek.ssl.subject.C,zeek.ssl.subject.O,zeek.ssl.subject.OU,zeek.ssl.subject.ST,zeek.ssl.subject.SN,zeek.ssl.subject.L,zeek.ssl.subject.GN,zeek.ssl.subject.pseudonym,zeek.ssl.subject.serialNumber,zeek.ssl.subject.title,zeek.ssl.subject.initials,zeek.ssl.subject.emailAddress,zeek.ssl.subject.description,zeek.ssl.subject.postalCode,zeek.ssl.subject.street,zeek.ssl.client_subject.CN,zeek.ssl.client_subject.C,zeek.ssl.client_subject.O,zeek.ssl.client_subject.OU,zeek.ssl.client_subject.ST,zeek.ssl.client_subject.SN,zeek.ssl.client_subject.L,zeek.ssl.client_subject.GN,zeek.ssl.client_subject.pseudonym,zeek.ssl.client_subject.serialNumber,zeek.ssl.client_subject.title,zeek.ssl.client_subject.initials,zeek.ssl.client_subject.emailAddress,zeek.ssl.issuer.CN,zeek.ssl.issuer.C,zeek.ssl.issuer.O,zeek.ssl.issuer.OU,zeek.ssl.issuer.ST,zeek.ssl.issuer.SN,zeek.ssl.issuer.L,zeek.ssl.issuer.DC,zeek.ssl.issuer.GN,zeek.ssl.issuer.pseudonym,zeek.ssl.issuer.serialNumber,zeek.ssl.issuer.title,zeek.ssl.issuer.initials,zeek.ssl.issuer.emailAddress,zeek.ssl.client_issuer.CN,zeek.ssl.client_issuer.C,zeek.ssl.client_issuer.O,zeek.ssl.client_issuer.OU,zeek.ssl.client_issuer.ST,zeek.ssl.client_issuer.SN,zeek.ssl.client_issuer.L,zeek.ssl.client_issuer.DC,zeek.ssl.client_issuer.GN,zeek.ssl.client_issuer.pseudonym,zeek.ssl.client_issuer.serialNumber,zeek.ssl.client_issuer.title,zeek.ssl.client_issuer.initials,zeek.ssl.client_issuer.emailAddress,zeek.ssl.validation_status o_zeek_stun=require:zeek.stun;title:Zeek stun.log;fields:zeek.stun.trans_id,zeek.stun.method,zeek.stun.class,zeek.stun.attr_type,zeek.stun.attr_val @@ -2681,7 +2681,7 @@ o_zeek_tds_rpc=require:zeek.tds_rpc;title:Zeek tds_rpc.log;fields:zeek.tds_rpc.p o_zeek_tds_sql_batch=require:zeek.tds_sql_batch;title:Zeek tds_sql_batch.log;fields:zeek.tds_sql_batch.header_type,zeek.tds_sql_batch.query o_zeek_tftp=require:zeek.tftp;title:Zeek tftp.log;fields:zeek.tftp.block_acked,zeek.tftp.block_sent,zeek.tftp.error_code,zeek.tftp.error_msg,zeek.tftp.fname,zeek.tftp.mode,zeek.tftp.size,zeek.tftp.uid_data,zeek.tftp.wrq o_zeek_tunnel=require:zeek.tunnel;title:Zeek tunnel.log;fields:zeek.tunnel.tunnel_type,zeek.tunnel.action -o_zeek_weird=require:zeek.weird;title:Zeek weird.log;fields:rule.name,zeek.weird.addl,zeek.weird.notice +o_zeek_weird=require:zeek.weird;title:Zeek weird.log;fields:rule.name,zeek.weird.addl,zeek.weird.notice,zeek.weird.source o_zeek_wireguard=require:zeek.wireguard;title:Zeek wireguard.log;fields:zeek.wireguard.established,zeek.wireguard.initiations,zeek.wireguard.responses o_zeek_x509=require:zeek.x509;title:Zeek x509.log;fields:zeek.x509.certificate_version,zeek.x509.certificate_serial,zeek.x509.certificate_subject.CN,zeek.x509.certificate_subject.C,zeek.x509.certificate_subject.O,zeek.x509.certificate_subject.OU,zeek.x509.certificate_subject.ST,zeek.x509.certificate_subject.SN,zeek.x509.certificate_subject.L,zeek.x509.certificate_subject.DC,zeek.x509.certificate_subject.GN,zeek.x509.certificate_subject.pseudonym,zeek.x509.certificate_subject.serialNumber,zeek.x509.certificate_subject.title,zeek.x509.certificate_subject.initials,zeek.x509.certificate_subject.emailAddress,zeek.x509.certificate_subject.description,zeek.x509.certificate_subject.postalCode,zeek.x509.certificate_subject.street,zeek.x509.certificate_issuer.CN,zeek.x509.certificate_issuer.DC,zeek.x509.certificate_issuer.C,zeek.x509.certificate_issuer.O,zeek.x509.certificate_issuer.OU,zeek.x509.certificate_issuer.ST,zeek.x509.certificate_issuer.SN,zeek.x509.certificate_issuer.L,zeek.x509.certificate_issuer.GN,zeek.x509.certificate_issuer.pseudonym,zeek.x509.certificate_issuer.serialNumber,zeek.x509.certificate_issuer.title,zeek.x509.certificate_issuer.initials,zeek.x509.certificate_issuer.emailAddress,zeek.x509.certificate_not_valid_before,zeek.x509.certificate_not_valid_after,zeek.x509.certificate_key_alg,zeek.x509.certificate_sig_alg,zeek.x509.certificate_key_type,zeek.x509.certificate_key_length,zeek.x509.certificate_exponent,zeek.x509.certificate_curve,zeek.x509.client_cert,zeek.x509.fingerprint,zeek.x509.host_cert,zeek.x509.san_dns,zeek.x509.san_uri,zeek.x509.san_email,zeek.x509.san_ip,zeek.x509.basic_constraints_ca,zeek.x509.basic_constraints_path_len p_suricata=require:suricata.timestamp;title:Suricata Common Fields;fields:suricata.action,suricata.app_proto_expected,suricata.app_proto_orig,suricata.app_proto_tc,suricata.app_proto_ts,suricata.category,suricata.flow_id,suricata.pcap_cnt,suricata.pcap_filename,suricata.severity,suricata.signature,suricata.stream,suricata.timestamp,suricata.tx_id diff --git a/arkime/wise/source.zeeklogs.js b/arkime/wise/source.zeeklogs.js index 9e30b2a30..1486b9242 100644 --- a/arkime/wise/source.zeeklogs.js +++ b/arkime/wise/source.zeeklogs.js @@ -2105,6 +2105,7 @@ class MalcolmSource extends WISESource { "zeek.uid", "zeek.weird.addl", "zeek.weird.notice", + "zeek.weird.source", "zeek.wireguard.established", "zeek.wireguard.initiations", "zeek.wireguard.receiver_index", diff --git a/dashboards/templates/composable/component/zeek.json b/dashboards/templates/composable/component/zeek.json index 30f8d0f4e..75da88606 100644 --- a/dashboards/templates/composable/component/zeek.json +++ b/dashboards/templates/composable/component/zeek.json @@ -453,6 +453,7 @@ "zeek.software.name": { "type": "keyword", "ignore_above": 256, "fields": { "text": { "type": "text" } } }, "zeek.software.software_type": { "type": "keyword" }, "zeek.software.unparsed_version": { "type": "keyword", "ignore_above": 1024 }, + "zeek.software.url": { "type": "keyword", "ignore_above": 16384, "fields": { "text": { "type": "text" } } }, "zeek.software.version_addl": { "type": "keyword", "ignore_above": 1024 }, "zeek.software.version_major": { "type": "integer" }, "zeek.software.version_minor": { "type": "integer" }, @@ -592,6 +593,7 @@ "zeek.uid": { "type": "keyword" }, "zeek.weird.addl": { "type": "keyword", "doc_values": false, "ignore_above": 16384, "fields": { "text": { "type": "text", "norms": false } } }, "zeek.weird.notice": { "type": "keyword" }, + "zeek.weird.source": { "type": "keyword" }, "zeek.wireguard.established": { "type": "keyword" }, "zeek.wireguard.initiations": { "type": "integer" }, "zeek.wireguard.receiver_index": { "type": "keyword" }, diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index 37e2d84c3..7d63dc04f 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -2188,7 +2188,7 @@ filter { id => "dissect_zeek_modbus_read_write_multiple_registers" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][`]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][write_start_address]} %{[zeek_cols][write_registers]} %{[zeek_cols][read_start_address]} %{[zeek_cols][read_quantity]} %{[zeek_cols][read_registers]}" + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_id]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][write_start_address]} %{[zeek_cols][write_registers]} %{[zeek_cols][read_start_address]} %{[zeek_cols][read_quantity]} %{[zeek_cols][read_registers]}" } } if ("_dissectfailure" in [tags]) { @@ -3001,10 +3001,10 @@ filter { if ("_jsonparsesuccess" in [tags]) { mutate { id => "mutate_rename_zeek_json_smb_cmd_referenced_file_fields" - rename => { "[zeek_cols][referenced_file.id.orig_h]" => "[zeek_cols][referenced_file.orig_h]" } - rename => { "[zeek_cols][referenced_file.id.orig_p]" => "[zeek_cols][referenced_file.orig_p]" } - rename => { "[zeek_cols][referenced_file.id.resp_h]" => "[zeek_cols][referenced_file.resp_h]" } - rename => { "[zeek_cols][referenced_file.id.resp_p]" => "[zeek_cols][referenced_file.resp_p]" } + rename => { "[zeek_cols][referenced_file.id.orig_h]" => "[zeek_cols][referenced_file][orig_h]" } + rename => { "[zeek_cols][referenced_file.id.orig_p]" => "[zeek_cols][referenced_file][orig_p]" } + rename => { "[zeek_cols][referenced_file.id.resp_h]" => "[zeek_cols][referenced_file][resp_h]" } + rename => { "[zeek_cols][referenced_file.id.resp_p]" => "[zeek_cols][referenced_file][resp_p]" } } } else { @@ -4018,8 +4018,8 @@ filter { } } mutate { id => "mutate_split_zeek_x509_san_ip" - split => { "[zeek][x509][san_ip]" => "," - "[zeek][x509][fingerprint]" => "," } } + split => { "[zeek_cols][san_ip]" => "," + "[zeek_cols][fingerprint]" => "," } } } mutate { id => "mutate_add_fields_zeek_x509" From 88d81474d8aee0814dad941d21b5bf9cde611d25 Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 21 Feb 2024 15:55:19 -0700 Subject: [PATCH 22/79] for supporting JSON logs from Zeek (idaholab/Malcolm#65); getting closer --- arkime/etc/config.ini | 7 ++++++- arkime/wise/source.zeeklogs.js | 15 ++++++++++----- .../templates/composable/component/zeek.json | 4 ++++ docs/contributing-logstash.md | 4 ++-- logstash/pipelines/zeek/12_zeek_mutate.conf | 8 +++++++- logstash/pipelines/zeek/13_zeek_normalize.conf | 4 ++++ 6 files changed, 33 insertions(+), 9 deletions(-) diff --git a/arkime/etc/config.ini b/arkime/etc/config.ini index 1dedadba3..01a0733e8 100644 --- a/arkime/etc/config.ini +++ b/arkime/etc/config.ini @@ -1628,6 +1628,11 @@ zeek.smb_files.times_changed=db:zeek.smb_files.times_changed;group:zeek_smb;kind zeek.smb_files.data_offset_req=db:zeek.smb_files.data_offset_req;group:zeek_smb;kind:integer;viewerOnly:true;friendly:Data Offset Requested;help:Data Offset Requested zeek.smb_files.data_len_req=db:zeek.smb_files.data_len_req;group:zeek_smb;kind:integer;viewerOnly:true;friendly:Data Length Requested;help:Data Length Requested zeek.smb_files.data_len_rsp=db:zeek.smb_files.data_len_rsp;group:zeek_smb;kind:integer;viewerOnly:true;friendly:Data Length In Response;help:Data Length In Response +zeek.smb_files.ts=db:zeek.smb_files.ts;group:zeek_smb;kind:termfield;viewerOnly:true;friendly:Zeek Timestamp for Referenced File;help:Zeek Timestamp for Referenced File +zeek.smb_files.orig_h=db:zeek.smb_files.orig_h;group:zeek_smb;kind:termfield;viewerOnly:true;friendly:Originating IP for Referenced File;help:Originating IP for Referenced File +zeek.smb_files.orig_p=db:zeek.smb_files.orig_p;group:zeek_smb;kind:integer;viewerOnly:true;friendly:Originating Port for Referenced File;help:Originating Port for Referenced File +zeek.smb_files.resp_h=db:zeek.smb_files.resp_h;group:zeek_smb;kind:termfield;viewerOnly:true;friendly:Responding IP for Referenced File;help:Responding IP for Referenced File +zeek.smb_files.resp_p=db:zeek.smb_files.resp_p;group:zeek_smb;kind:integer;viewerOnly:true;friendly:Responding Port for Referenced File;help:Responding Port for Referenced File # smb_mapping.log # https://docs.zeek.org/en/stable/scripts/base/protocols/smb/main.zeek.html#type-SMB::TreeInfo @@ -2658,7 +2663,7 @@ o_zeek_s7comm_upload_download=require:zeek.s7comm_upload_download;title:Zeek s7c o_zeek_signatures=require:zeek.signatures;title:Zeek signatures.log;fields:event.module,rule.category,rule.name,vulnerability.category,vulnerability.enumeration,vulnerability.id,zeek.signatures.sub_message,zeek.signatures.signature_count,zeek.signatures.host_count o_zeek_sip=require:zeek.sip;title:Zeek sip.log;fields:zeek.sip.trans_depth,zeek.sip.method,zeek.sip.uri,zeek.sip.date,zeek.sip.request_from,zeek.sip.request_to,zeek.sip.response_from,zeek.sip.response_to,zeek.sip.reply_to,zeek.sip.call_id,zeek.sip.seq,zeek.sip.subject,zeek.sip.request_path,zeek.sip.response_path,zeek.sip.user_agent,zeek.sip.status_code,zeek.sip.status_msg,zeek.sip.warning,zeek.sip.request_body_len,zeek.sip.response_body_len,zeek.sip.content_type,zeek.sip.version o_zeek_smb_cmd=require:zeek.smb_cmd;title:Zeek smb_cmd.log;fields:zeek.smb_cmd.command,zeek.smb_cmd.sub_command,zeek.smb_cmd.argument,zeek.smb_cmd.status,zeek.smb_cmd.rtt,zeek.smb_cmd.version,zeek.smb_cmd.user,zeek.smb_cmd.tree,zeek.smb_cmd.tree_service -o_zeek_smb_files=require:zeek.smb_files;title:Zeek smb_files.log;fields:zeek.smb_files.action,zeek.smb_files.path,zeek.smb_files.name,zeek.smb_files.size,zeek.smb_files.prev_name,zeek.smb_files.times_modified,zeek.smb_files.times_accessed,zeek.smb_files.times_created,zeek.smb_files.times_changed,zeek.smb_files.data_offset_req,zeek.smb_files.data_len_req,zeek.smb_files.data_len_rsp +o_zeek_smb_files=require:zeek.smb_files;title:Zeek smb_files.log;fields:zeek.smb_files.action,zeek.smb_files.path,zeek.smb_files.name,zeek.smb_files.size,zeek.smb_files.prev_name,zeek.smb_files.times_modified,zeek.smb_files.times_accessed,zeek.smb_files.times_created,zeek.smb_files.times_changed,zeek.smb_files.data_offset_req,zeek.smb_files.data_len_req,zeek.smb_files.data_len_rsp,zeek.smb_files.ts,zeek.smb_files.orig_h,zeek.smb_files.orig_p,zeek.smb_files.resp_h,zeek.smb_files.resp_p o_zeek_smb_mapping=require:zeek.smb_mapping;title:Zeek smb_mapping.log;fields:zeek.smb_mapping.path,zeek.smb_mapping.resource_type,zeek.smb_mapping.native_file_system,zeek.smb_mapping.share_type o_zeek_smtp=require:zeek.smtp;title:Zeek smtp.log;fields:zeek.smtp.trans_depth,zeek.smtp.helo,zeek.smtp.mailfrom,zeek.smtp.rcptto,zeek.smtp.date,zeek.smtp.from,zeek.smtp.to,zeek.smtp.cc,zeek.smtp.reply_to,zeek.smtp.msg_id,zeek.smtp.in_reply_to,zeek.smtp.subject,zeek.smtp.x_originating_ip,zeek.smtp.first_received,zeek.smtp.second_received,zeek.smtp.last_reply,zeek.smtp.last_reply_code,zeek.smtp.last_reply_msg,zeek.smtp.path,zeek.smtp.user_agent,zeek.smtp.tls,zeek.smtp.is_webmail o_zeek_snmp=require:zeek.snmp;title:Zeek snmp.log;fields:zeek.snmp.duration,zeek.snmp.version,zeek.snmp.community,zeek.snmp.get_requests,zeek.snmp.get_bulk_requests,zeek.snmp.get_responses,zeek.snmp.set_requests,zeek.snmp.display_string,zeek.snmp.up_since diff --git a/arkime/wise/source.zeeklogs.js b/arkime/wise/source.zeeklogs.js index 1486b9242..8f91bf005 100644 --- a/arkime/wise/source.zeeklogs.js +++ b/arkime/wise/source.zeeklogs.js @@ -749,15 +749,15 @@ class MalcolmSource extends WISESource { "vulnerability.reference", "vulnerability.scanner.vendor", "zeek.bacnet.bvlc_function", - "zeek.bacnet.invoke_id", "zeek.bacnet.instance_number", + "zeek.bacnet.invoke_id", "zeek.bacnet.pdu_service", "zeek.bacnet.pdu_type", "zeek.bacnet.result_code", - "zeek.bacnet_device_control.time_duration", "zeek.bacnet_device_control.device_state", "zeek.bacnet_device_control.result", "zeek.bacnet_device_control.result_code", + "zeek.bacnet_device_control.time_duration", "zeek.bacnet_discovery.object_name", "zeek.bacnet_discovery.object_type", "zeek.bacnet_discovery.range", @@ -1114,20 +1114,20 @@ class MalcolmSource extends WISESource { "zeek.login.success", "zeek.modbus.exception", "zeek.modbus.func", + "zeek.modbus.mei_type", "zeek.modbus.network_direction", "zeek.modbus.trans_id", "zeek.modbus.unit_id", - "zeek.modbus.mei_type", "zeek.modbus_detailed.address", "zeek.modbus_detailed.quantity", "zeek.modbus_detailed.values", "zeek.modbus_mask_write_register.and_mask", "zeek.modbus_mask_write_register.or_mask", - "zeek.modbus_read_device_identification.conformity_level_code", "zeek.modbus_read_device_identification.conformity_level", + "zeek.modbus_read_device_identification.conformity_level_code", "zeek.modbus_read_device_identification.device_id_code", - "zeek.modbus_read_device_identification.object_id_code", "zeek.modbus_read_device_identification.object_id", + "zeek.modbus_read_device_identification.object_id_code", "zeek.modbus_read_device_identification.object_value", "zeek.modbus_read_write_multiple_registers.read_quantity", "zeek.modbus_read_write_multiple_registers.read_registers", @@ -1826,13 +1826,18 @@ class MalcolmSource extends WISESource { "zeek.smb_files.data_len_rsp", "zeek.smb_files.data_offset_req", "zeek.smb_files.name", + "zeek.smb_files.orig_h", + "zeek.smb_files.orig_p", "zeek.smb_files.path", "zeek.smb_files.prev_name", + "zeek.smb_files.resp_h", + "zeek.smb_files.resp_p", "zeek.smb_files.size", "zeek.smb_files.times_accessed", "zeek.smb_files.times_changed", "zeek.smb_files.times_created", "zeek.smb_files.times_modified", + "zeek.smb_files.ts", "zeek.smb_mapping.native_file_system", "zeek.smb_mapping.path", "zeek.smb_mapping.resource_type", diff --git a/dashboards/templates/composable/component/zeek.json b/dashboards/templates/composable/component/zeek.json index 75da88606..14273bb7c 100644 --- a/dashboards/templates/composable/component/zeek.json +++ b/dashboards/templates/composable/component/zeek.json @@ -407,6 +407,10 @@ "zeek.smb_files.times_created": { "type": "date" }, "zeek.smb_files.times_modified": { "type": "date" }, "zeek.smb_files.ts": { "type": "date" }, + "zeek.smb_files.orig_h": { "type": "ip" }, + "zeek.smb_files.orig_p": { "type": "integer" }, + "zeek.smb_files.resp_h": { "type": "ip" }, + "zeek.smb_files.resp_p": { "type": "integer" }, "zeek.smb_mapping.native_file_system": { "type": "keyword" }, "zeek.smb_mapping.path": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, "zeek.smb_mapping.resource_type": { "type": "keyword" }, diff --git a/docs/contributing-logstash.md b/docs/contributing-logstash.md index b37d5650d..c9cfa9d5f 100644 --- a/docs/contributing-logstash.md +++ b/docs/contributing-logstash.md @@ -38,8 +38,8 @@ The following modifications must be made in order for Malcolm to parse new Zeek * Follow patterns for existing log files as an example * For common Zeek fields such as the `id` four-tuple, timestamp, etc., use the same convention used by existing Zeek logs in that file (e.g., `ts`, `uid`, `orig_h`, `orig_p`, `resp_h`, `resp_p`) * Take care, especially when copy-pasting filter code, the Zeek delimiter isn't modified from a tab character to a space character (see "*zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP*" warnings in that file) -1. If necessary, perform log normalization in [`logstash/pipelines/zeek/12_zeek_normalize.conf`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/logstash/pipelines/zeek/12_zeek_normalize.conf) for values such as action (`event.action`), result (`event.result`), application protocol version (`network.protocol_version`), etc. -1. If necessary, define conversions for floating point or integer values in [`logstash/pipelines/zeek/11_zeek_parse.conf`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/logstash/pipelines/zeek/14_zeek_convert.conf) +1. If necessary, perform log normalization in [`logstash/pipelines/zeek/13_zeek_normalize.conf`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/logstash/pipelines/zeek/13_zeek_normalize.conf) for values such as action (`event.action`), result (`event.result`), application protocol version (`network.protocol_version`), etc. +1. If necessary, define conversions for floating point or integer values in [`logstash/pipelines/zeek/14_zeek_convert.conf`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/logstash/pipelines/zeek/14_zeek_convert.conf) 1. Identify the new fields and add them as described in [Adding new log fields](contributing-new-log-fields.md#NewFields) The script [`scripts/zeek_script_to_malcolm_boilerplate.py`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/scripts/zeek_script_to_malcolm_boilerplate.py) may help by autogenerating these filters. diff --git a/logstash/pipelines/zeek/12_zeek_mutate.conf b/logstash/pipelines/zeek/12_zeek_mutate.conf index 85a026b67..4ea9f12b0 100644 --- a/logstash/pipelines/zeek/12_zeek_mutate.conf +++ b/logstash/pipelines/zeek/12_zeek_mutate.conf @@ -1644,9 +1644,15 @@ filter { } } - # collect referenced file FUID(s) at parent level (here rather than in 12_zeek_normalize.conf because + # collect referenced file UIDs(s)/FUID(s) at parent level (here rather than in 13_zeek_normalize.conf because # this would have already been done as a root-level fuid array in the main "rename" above if we # had not had to move it up a level just now) + if ([zeek][smb_files][uid]) { + mutate { id => "mutate_merge_zeek_smb_files_uid" + merge => { "[zeek][uid]" => "[zeek][smb_files][uid]" } } + mutate { id => "mutate_remove_zeek_smb_files_uid" + remove_field => [ "[zeek][smb_files][uid]" ] } + } if ([zeek][smb_files][fuid]) { mutate { id => "mutate_merge_zeek_smb_files_fuid" merge => { "[zeek][fuid]" => "[zeek][smb_files][fuid]" } } diff --git a/logstash/pipelines/zeek/13_zeek_normalize.conf b/logstash/pipelines/zeek/13_zeek_normalize.conf index a57d65cb3..3c533bdfc 100644 --- a/logstash/pipelines/zeek/13_zeek_normalize.conf +++ b/logstash/pipelines/zeek/13_zeek_normalize.conf @@ -1450,6 +1450,10 @@ filter { merge => { "[related][ip]" => "[zeek][ospf][fwd_addrs]" } } } if ([zeek][ospf][dest_router_id]) { mutate { id => "mutate_merge_field_related_ip_zeek_ospf_dest_router_id" merge => { "[related][ip]" => "[zeek][ospf][dest_router_id]" } } } + if ([zeek][smb_files][orig_h]) { mutate { id => "mutate_merge_field_related_ip_zeek_smb_files_orig_h" + merge => { "[related][ip]" => "[zeek][smb_files][orig_h]" } } } + if ([zeek][smb_files][resp_h]) { mutate { id => "mutate_merge_field_related_ip_zeek_smb_files_resp_h" + merge => { "[related][ip]" => "[zeek][smb_files][resp_h]" } } } if ([zeek][radius][framed_addr]) { mutate { id => "mutate_merge_field_related_ip_zeek_radius_framed_addr" merge => { "[related][ip]" => "[zeek][radius][framed_addr]" } } } if ([zeek][smtp][path]) { mutate { id => "mutate_merge_field_related_ip_zeek_smtp_path" From 94f8a221f4fff41da83ba6367a19b07d7cdb8c59 Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 21 Feb 2024 16:27:01 -0700 Subject: [PATCH 23/79] logo tweaks --- dashboards/opensearch_dashboards.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboards/opensearch_dashboards.yml b/dashboards/opensearch_dashboards.yml index 884c39533..2a7f64fe3 100644 --- a/dashboards/opensearch_dashboards.yml +++ b/dashboards/opensearch_dashboards.yml @@ -18,8 +18,8 @@ opensearchDashboards.branding: applicationTitle: "Malcolm Dashboards" useExpandedHeader: false logo: - defaultUrl: "/assets/img/Malcolm.svg" - darkModeUrl: "/assets/img/malcolm_logo.svg" + defaultUrl: "/assets/img/Malcolm_outline_banner_dark.png" + darkModeUrl: "/assets/img/Malcolm_outline_banner_dark.png" mark: defaultUrl: "/assets/img/icon.png" darkModeUrl: "/assets/img/icon_dark.png" From 136292780df1b249c41c4674fe9dea9a53fac59f Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 21 Feb 2024 22:42:49 -0700 Subject: [PATCH 24/79] bump elasticsearcy-py to 8.12.1 --- api/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/requirements.txt b/api/requirements.txt index 3f95576d4..462a38cc5 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -5,5 +5,5 @@ opensearch-py==2.4.2 requests==2.31.0 regex==2022.3.2 dateparser==1.1.1 -elasticsearch==8.12.0 +elasticsearch==8.12.1 elasticsearch-dsl==8.12.0 \ No newline at end of file From 8701cf01e7484d8e0a08617aab21d559f0a51491 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 22 Feb 2024 08:03:21 -0700 Subject: [PATCH 25/79] fix idaholab/Malcolm#427, fix links on extracted-files page --- .../scripts/extracted_files_http_server.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/file-monitor/scripts/extracted_files_http_server.py b/file-monitor/scripts/extracted_files_http_server.py index 2f10d9a97..3eaab24ff 100755 --- a/file-monitor/scripts/extracted_files_http_server.py +++ b/file-monitor/scripts/extracted_files_http_server.py @@ -103,7 +103,6 @@ def do_GET(self): # with doc.head: - base(target="_blank") meta(charset="utf-8") meta(name="viewport", content="width=device-width, initial-scale=1, shrink-to-fit=no") link(rel="icon", href=f"{args.assetsDirRespReplacer}favicon.ico", type="image/x-icon") @@ -243,6 +242,7 @@ def do_GET(self): a( fileinfo, href=f'https://www.iana.org/assignments/media-types/{fileinfo}', + target="_blank", ), ) if args.magic @@ -264,6 +264,7 @@ def do_GET(self): a( fid, href=f'/arkime/idark2dash/filter?start={timestampStartFilterStr}&stop={tomorrowStr}&field=event.id&value={fid}', + target="_blank", ) for fid in fids ], @@ -298,19 +299,24 @@ def do_GET(self): ) with div(cls="col-lg-6 h-100 text-center text-lg-end my-auto").add(ul(cls="list-inline mb-0")): - li(cls="list-inline-item").add(a(href=f'/')).add(i(cls="bi bi-house fs-3", title="Malcolm")) - li(cls="list-inline-item").add(a(href=f'/readme/')).add( + li(cls="list-inline-item").add(a(href=f'/', target="_blank")).add( + i(cls="bi bi-house fs-3", title="Malcolm") + ) + li(cls="list-inline-item").add(a(href=f'/readme/', target="_blank")).add( i(cls="bi bi-question-circle fs-3", title="Documentation") ) li(cls="list-inline-item").add( - a(href=f'/dashboards/app/dashboards#/view/9ee51f94-3316-4fc5-bd89-93a52af69714') + a( + href=f'/dashboards/app/dashboards#/view/9ee51f94-3316-4fc5-bd89-93a52af69714', + target="_blank", + ) ).add(i(cls="bi bi-bar-chart-line fs-3", title="Dashboards")) - li(cls="list-inline-item").add(a(href=f'/arkime/sessions/')).add( + li(cls="list-inline-item").add(a(href=f'/arkime/sessions/', target="_blank")).add( i(cls="bi bi-table fs-3", title="Arkime") ) - li(cls="list-inline-item").add(a(href=f'https://github.com/idaholab/Malcolm/')).add( - i(cls="bi-github fs-3", title="GitHub") - ) + li(cls="list-inline-item").add( + a(href=f'https://github.com/idaholab/Malcolm/', target="_blank") + ).add(i(cls="bi-github fs-3", title="GitHub")) script(type="text/javascript", src=f"{args.assetsDirRespReplacer}js/bootstrap.bundle.min.js") script(type="text/javascript", src=f"{args.assetsDirRespReplacer}js/scripts.js") From 862d3d58cbb61dae803a0f514dadbc838f6d674c Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 22 Feb 2024 11:19:16 -0700 Subject: [PATCH 26/79] bump logstash and beats to v8.12.2 --- Dockerfiles/filebeat.Dockerfile | 2 +- Dockerfiles/logstash.Dockerfile | 2 +- hedgehog-iso/build.sh | 2 +- hedgehog-raspi/sensor_install.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index 65b75e4e1..3c9502800 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/beats/filebeat-oss:8.12.1 +FROM docker.elastic.co/beats/filebeat-oss:8.12.2 # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/Dockerfiles/logstash.Dockerfile b/Dockerfiles/logstash.Dockerfile index 9e49f9deb..5505fb201 100644 --- a/Dockerfiles/logstash.Dockerfile +++ b/Dockerfiles/logstash.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/logstash/logstash-oss:8.12.1 +FROM docker.elastic.co/logstash/logstash-oss:8.12.2 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' diff --git a/hedgehog-iso/build.sh b/hedgehog-iso/build.sh index 67c0dfe5e..16f2f4f92 100755 --- a/hedgehog-iso/build.sh +++ b/hedgehog-iso/build.sh @@ -5,7 +5,7 @@ IMAGE_PUBLISHER=idaholab IMAGE_VERSION=1.0.0 IMAGE_DISTRIBUTION=bookworm -BEATS_VER="8.12.1" +BEATS_VER="8.12.2" BEATS_OSS="-oss" BUILD_ERROR_CODE=1 diff --git a/hedgehog-raspi/sensor_install.sh b/hedgehog-raspi/sensor_install.sh index d15225bcd..9ddd4f34f 100644 --- a/hedgehog-raspi/sensor_install.sh +++ b/hedgehog-raspi/sensor_install.sh @@ -32,7 +32,7 @@ SHARED_DIR='/opt/buildshared' WORK_DIR="$(mktemp -d -t hedgehog-XXXXXX)" SENSOR_DIR='/opt/sensor' -BEATS_VER="8.12.1" +BEATS_VER="8.12.2" BEATS_OSS="-oss" # Option to build from sources if desired From 594c641b2e89aae0f3487d7588981c415b76cf67 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 22 Feb 2024 11:27:06 -0700 Subject: [PATCH 27/79] for supporting JSON logs from Zeek (idaholab/Malcolm#65); getting closer --- arkime/etc/config.ini | 10 +++++----- arkime/wise/source.zeeklogs.js | 8 ++++---- dashboards/templates/composable/component/zeek.json | 8 ++++---- logstash/pipelines/zeek/11_zeek_parse.conf | 12 +++++++++--- logstash/pipelines/zeek/13_zeek_normalize.conf | 8 ++++---- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/arkime/etc/config.ini b/arkime/etc/config.ini index 01a0733e8..ef02d8239 100644 --- a/arkime/etc/config.ini +++ b/arkime/etc/config.ini @@ -523,10 +523,10 @@ zeek.ftp.mime_type=db:zeek.ftp.mime_type;group:zeek_ftp;kind:termfield;viewerOnl zeek.ftp.file_size=db:zeek.ftp.file_size;group:zeek_ftp;kind:integer;viewerOnly:true;friendly:File Size;help:File Size zeek.ftp.reply_code=db:zeek.ftp.reply_code;group:zeek_ftp;kind:integer;viewerOnly:true;friendly:Reply Code;help:Reply Code zeek.ftp.reply_msg=db:zeek.ftp.reply_msg;group:zeek_ftp;kind:termfield;viewerOnly:true;friendly:Reply;help:Reply -zeek.ftp.data_channel_passive=db:zeek.ftp.data_channel_passive;group:zeek_ftp;kind:termfield;viewerOnly:true;friendly:Passive;help:Passive -zeek.ftp.data_channel_orig_h=db:zeek.ftp.data_channel_orig_h;group:zeek_ftp;kind:termfield;viewerOnly:true;friendly:Data Originating Host;help:Data Originating Host -zeek.ftp.data_channel_resp_h=db:zeek.ftp.data_channel_resp_h;group:zeek_ftp;kind:termfield;viewerOnly:true;friendly:Data Responding Host;help:Data Responding Host -zeek.ftp.data_channel_resp_p=db:zeek.ftp.data_channel_resp_p;group:zeek_ftp;kind:integer;viewerOnly:true;friendly:Data Responding Port;help:Data Responding Port +zeek.ftp.data_channel.passive=db:zeek.ftp.data_channel.passive;group:zeek_ftp;kind:termfield;viewerOnly:true;friendly:Passive;help:Passive +zeek.ftp.data_channel.orig_h=db:zeek.ftp.data_channel.orig_h;group:zeek_ftp;kind:termfield;viewerOnly:true;friendly:Data Originating Host;help:Data Originating Host +zeek.ftp.data_channel.resp_h=db:zeek.ftp.data_channel.resp_h;group:zeek_ftp;kind:termfield;viewerOnly:true;friendly:Data Responding Host;help:Data Responding Host +zeek.ftp.data_channel.resp_p=db:zeek.ftp.data_channel.resp_p;group:zeek_ftp;kind:integer;viewerOnly:true;friendly:Data Responding Port;help:Data Responding Port # genisys.log # https://github.com/cisagov/icsnpp-genisys @@ -2622,7 +2622,7 @@ o_zeek_ecat_registers=require:zeek.ecat_registers;title:Zeek ecat_registers.log; o_zeek_ecat_soe_info=require:zeek.ecat_soe_info;title:Zeek ecat_soe_info.log;fields:zeek.ecat_soe_info.opcode,zeek.ecat_soe_info.incomplete,zeek.ecat_soe_info.error,zeek.ecat_soe_info.drive_num,zeek.ecat_soe_info.element,zeek.ecat_soe_info.index o_zeek_enip=require:zeek.enip;title:Zeek enip.log;fields:zeek.enip.enip_command,zeek.enip.enip_command_code,zeek.enip.length,zeek.enip.session_handle,zeek.enip.enip_status,zeek.enip.sender_context,zeek.enip.options o_zeek_files=require:zeek.files;title:Zeek files.log;fields:zeek.files.tx_hosts,zeek.files.rx_hosts,zeek.files.conn_uids,zeek.files.source,zeek.files.depth,zeek.files.analyzers,zeek.files.mime_type,zeek.files.filename,zeek.files.ftime,zeek.files.duration,zeek.files.local_orig,zeek.files.seen_bytes,zeek.files.total_bytes,zeek.files.missing_bytes,zeek.files.overflow_bytes,zeek.files.timedout,zeek.files.parent_fuid,zeek.files.md5,zeek.files.sha1,zeek.files.sha256,zeek.files.extracted,zeek.files.extracted_cutoff,zeek.files.extracted_size -o_zeek_ftp=require:zeek.ftp;title:Zeek ftp.log;fields:zeek.ftp.command,zeek.ftp.arg,zeek.ftp.mime_type,zeek.ftp.file_size,zeek.ftp.reply_code,zeek.ftp.reply_msg,zeek.ftp.data_channel_passive,zeek.ftp.data_channel_orig_h,zeek.ftp.data_channel_resp_h,zeek.ftp.data_channel_resp_p +o_zeek_ftp=require:zeek.ftp;title:Zeek ftp.log;fields:zeek.ftp.command,zeek.ftp.arg,zeek.ftp.mime_type,zeek.ftp.file_size,zeek.ftp.reply_code,zeek.ftp.reply_msg,zeek.ftp.data_channel.passive,zeek.ftp.data_channel.orig_h,zeek.ftp.data_channel.resp_h,zeek.ftp.data_channel.resp_p o_zeek_genisys=require:zeek.genisys;title:Zeek genisys.log;fields:zeek.genisys.header,zeek.genisys.server,zeek.genisys.direction,zeek.genisys.crc_transmitted,zeek.genisys.crc_calculated,zeek.genisys.payload.address,zeek.genisys.payload.data o_zeek_gquic=require:zeek.gquic;title:Zeek gquic.log;fields:zeek.gquic.version,zeek.gquic.server_name,zeek.gquic.user_agent,zeek.gquic.tag_count,zeek.gquic.cyu,zeek.gquic.cyutags o_zeek_http=require:zeek.http;title:Zeek http.log;fields:zeek.http.trans_depth,zeek.http.method,zeek.http.host,zeek.http.uri,zeek.http.origin,zeek.http.post_password_plain,zeek.http.post_username,zeek.http.referrer,zeek.http.version,zeek.http.user_agent,zeek.http.request_body_len,zeek.http.response_body_len,zeek.http.status_code,zeek.http.status_msg,zeek.http.info_code,zeek.http.info_msg,zeek.http.tags,zeek.http.proxied,zeek.http.orig_fuids,zeek.http.orig_filenames,zeek.http.orig_mime_types,zeek.http.resp_fuids,zeek.http.resp_filenames,zeek.http.resp_mime_types diff --git a/arkime/wise/source.zeeklogs.js b/arkime/wise/source.zeeklogs.js index 8f91bf005..80fa72faa 100644 --- a/arkime/wise/source.zeeklogs.js +++ b/arkime/wise/source.zeeklogs.js @@ -983,10 +983,10 @@ class MalcolmSource extends WISESource { "zeek.files.tx_hosts", "zeek.ftp.arg", "zeek.ftp.command", - "zeek.ftp.data_channel_orig_h", - "zeek.ftp.data_channel_passive", - "zeek.ftp.data_channel_resp_h", - "zeek.ftp.data_channel_resp_p", + "zeek.ftp.data_channel.orig_h", + "zeek.ftp.data_channel.passive", + "zeek.ftp.data_channel.resp_h", + "zeek.ftp.data_channel.resp_p", "zeek.ftp.file_size", "zeek.ftp.mime_type", "zeek.ftp.reply_code", diff --git a/dashboards/templates/composable/component/zeek.json b/dashboards/templates/composable/component/zeek.json index 14273bb7c..76b20aafd 100644 --- a/dashboards/templates/composable/component/zeek.json +++ b/dashboards/templates/composable/component/zeek.json @@ -84,10 +84,10 @@ "zeek.files.tx_hosts": { "type": "ip" }, "zeek.ftp.arg": { "type": "keyword" }, "zeek.ftp.command": { "type": "keyword" }, - "zeek.ftp.data_channel_orig_h": { "type": "ip" }, - "zeek.ftp.data_channel_passive": { "type": "keyword" }, - "zeek.ftp.data_channel_resp_h": { "type": "ip" }, - "zeek.ftp.data_channel_resp_p": { "type": "integer" }, + "zeek.ftp.data_channel.orig_h": { "type": "ip" }, + "zeek.ftp.data_channel.passive": { "type": "keyword" }, + "zeek.ftp.data_channel.resp_h": { "type": "ip" }, + "zeek.ftp.data_channel.resp_p": { "type": "integer" }, "zeek.ftp.file_size": { "type": "long" }, "zeek.ftp.mime_type": { "type": "keyword" }, "zeek.ftp.reply_code": { "type": "short" }, diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index 7d63dc04f..15926805c 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -1369,7 +1369,7 @@ filter { id => "dissect_zeek_ftp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][command]} %{[zeek_cols][arg]} %{[zeek_cols][mime_type]} %{[zeek_cols][file_size]} %{[zeek_cols][reply_code]} %{[zeek_cols][reply_msg]} %{[zeek_cols][data_channel_passive]} %{[zeek_cols][data_channel_orig_h]} %{[zeek_cols][data_channel_resp_h]} %{[zeek_cols][data_channel_resp_p]} %{[zeek_cols][fuid]}" + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][command]} %{[zeek_cols][arg]} %{[zeek_cols][mime_type]} %{[zeek_cols][file_size]} %{[zeek_cols][reply_code]} %{[zeek_cols][reply_msg]} %{[zeek_cols][data_channel][passive]} %{[zeek_cols][data_channel][orig_h]} %{[zeek_cols][data_channel][resp_h]} %{[zeek_cols][data_channel][resp_p]} %{[zeek_cols][fuid]}" } } if ("_dissectfailure" in [tags]) { @@ -1380,7 +1380,7 @@ filter { } ruby { id => "ruby_zip_zeek_ftp" - init => "@zeek_ftp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'password', 'command', 'arg', 'mime_type', 'file_size', 'reply_code', 'reply_msg', 'data_channel_passive', 'data_channel_orig_h', 'data_channel_resp_h', 'data_channel_resp_p', 'fuid' ]" + init => "@zeek_ftp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'password', 'command', 'arg', 'mime_type', 'file_size', 'reply_code', 'reply_msg', 'data_channel.passive', 'data_channel.orig_h', 'data_channel.resp_h', 'data_channel.resp_p', 'fuid' ]" code => "event.set('[zeek_cols]', @zeek_ftp_field_names.zip(event.get('[message]')).to_h)" } } @@ -1753,7 +1753,13 @@ filter { # known_hosts.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/conn/known-hosts.zeek.html#type-Known::HostsInfo - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_known_certs_fields" + rename => { "[zeek_cols][host]" => "[zeek_cols][orig_h]" } + } + + } else { dissect { id => "dissect_zeek_known_hosts" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP diff --git a/logstash/pipelines/zeek/13_zeek_normalize.conf b/logstash/pipelines/zeek/13_zeek_normalize.conf index 3c533bdfc..ddaf21874 100644 --- a/logstash/pipelines/zeek/13_zeek_normalize.conf +++ b/logstash/pipelines/zeek/13_zeek_normalize.conf @@ -1420,10 +1420,10 @@ filter { merge => { "[related][ip]" => "[zeek][files][rx_hosts]" } } } if ([zeek][files][tx_hosts]) { mutate { id => "mutate_merge_field_related_ip_zeek_files_tx_hosts" merge => { "[related][ip]" => "[zeek][files][tx_hosts]" } } } - if ([zeek][ftp][data_channel_orig_h]) { mutate { id => "mutate_merge_field_related_ip_zeek_ftp_data_channel_orig_h" - merge => { "[related][ip]" => "[zeek][ftp][data_channel_orig_h]" } } } - if ([zeek][ftp][data_channel_resp_h]) { mutate { id => "mutate_merge_field_related_ip_zeek_ftp_data_channel_resp_h" - merge => { "[related][ip]" => "[zeek][ftp][data_channel_resp_h]" } } } + if ([zeek][ftp][data_channel][orig_h]) { mutate { id => "mutate_merge_field_related_ip_zeek_ftp_data_channel_orig_h" + merge => { "[related][ip]" => "[zeek][ftp][data_channel][orig_h]" } } } + if ([zeek][ftp][data_channel][resp_h]) { mutate { id => "mutate_merge_field_related_ip_zeek_ftp_data_channel_resp_h" + merge => { "[related][ip]" => "[zeek][ftp][data_channel][resp_h]" } } } if ([zeek][notice][dst]) { mutate { id => "mutate_merge_field_related_ip_zeek_notice_dst" merge => { "[related][ip]" => "[zeek][notice][dst]" } } } if ([zeek][notice][src]) { mutate { id => "mutate_merge_field_related_ip_zeek_notice_src" From 1da36898be89ca3208bb1624d1399a114f88f16c Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 22 Feb 2024 11:38:48 -0700 Subject: [PATCH 28/79] update documentation --- docs/malcolm-config.md | 1 + docs/upload.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/malcolm-config.md b/docs/malcolm-config.md index 5f6bf38b8..9632d6aba 100644 --- a/docs/malcolm-config.md +++ b/docs/malcolm-config.md @@ -111,6 +111,7 @@ Although the configuration script automates many of the following configuration - `VTOT_API2_KEY` – used to specify a [VirusTotal Public API v.20](https://www.virustotal.com/en/documentation/public-api/) key, which, if specified, will be used to submit hashes of [Zeek-extracted files](file-scanning.md#ZeekFileExtraction) to VirusTotal - `ZEEK_AUTO_ANALYZE_PCAP_FILES` – if set to `true`, all PCAP files imported into Malcolm will automatically be analyzed by Zeek, and the resulting logs will also be imported (default `false`) - `ZEEK_AUTO_ANALYZE_PCAP_THREADS` – the number of threads available to Malcolm for analyzing Zeek logs (default `1`) + - `ZEEK_JSON` - whether Zeek should generate [JSON format logs](https://docs.zeek.org/en/master/log-formats.html#zeek-json-format-logs) (`true`) or [TSV format logs](https://docs.zeek.org/en/master/log-formats.html#zeek-tsv-format-logs) (`false`) - `ZEEK_DISABLE_…` - if set to `true`, each of these variables can be used to disable a certain Zeek function when it analyzes PCAP files (for example, setting `ZEEK_DISABLE_LOG_PASSWORDS` to `true` to disable logging of cleartext passwords) - `ZEEK_…_PORTS` - used to specify non-default ports to register certain Zeek analyzers (e.g., `ZEEK_SYNCHROPHASOR_PORTS` for the [ICSNPP-Synchrophasor analyzer](https://github.com/cisagov/icsnpp-synchrophasor/), `ZEEK_GENISYS_PORTS` for the [ICSNPP-Genisys analyzer](https://github.com/cisagov/icsnpp-genisys/), and `ZEEK_ENIP_PORTS` for the [ICSNPP-Ethernet/IP analyzer](https://github.com/cisagov/icsnpp-enip/)) formatted as a comma-separated list of [Zeek ports](https://docs.zeek.org/en/master/scripting/basics.html#port) (e.g., `12345/tcp` or `4041/tcp,4042/udp`) - `ZEEK_DISABLE_ICS_ALL` and `ZEEK_DISABLE_ICS_…` - if set to `true`, these variables can be used to disable Zeek's protocol analyzers for Operational Technology/Industrial Control Systems (OT/ICS) protocols diff --git a/docs/upload.md b/docs/upload.md index 4c3b8bb6f..4210402e7 100644 --- a/docs/upload.md +++ b/docs/upload.md @@ -15,6 +15,7 @@ The types of files supported are: * PCAP files (of mime type `application/vnd.tcpdump.pcap` or `application/x-pcapng`) - PCAPNG files are *partially* supported: Zeek is able to process PCAPNG files, but not all of Arkime's packet examination features work correctly * Zeek logs in archive files (`application/gzip`, `application/x-gzip`, `application/x-7z-compressed`, `application/x-bzip2`, `application/x-cpio`, `application/x-lzip`, `application/x-lzma`, `application/x-rar-compressed`, `application/x-tar`, `application/x-xz`, or `application/zip`) + - because log fields may differ depending on Zeek's configuration, users are recommended to use [Zeek JSON format logs](https://docs.zeek.org/en/master/log-formats.html#zeek-json-format-logs) when generating Zeek logs outside of Malcolm to later be uploaded to Malcolm for procesing - where the Zeek logs are found in the internal directory structure in the archive file does not matter Files uploaded via these methods are monitored and moved automatically to other directories for processing, generally within 1 minute of completion of the upload. From db6b783403de70690bfd523352f479d314a6b94a Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 22 Feb 2024 11:40:25 -0700 Subject: [PATCH 29/79] apparently with the latest logstash the order of -Xmx and -Xms matters, or you get a 'Initial heap size set to a larger value than the maximum heap size' error message --- arkime/arkime_regression_test_harness/docker-compose.yml | 2 +- config/logstash.env.example | 2 +- config/opensearch.env.example | 2 +- docs/malcolm-config.md | 4 ++-- scripts/install.py | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/arkime/arkime_regression_test_harness/docker-compose.yml b/arkime/arkime_regression_test_harness/docker-compose.yml index 1e112a7a6..8bd2d5f5b 100644 --- a/arkime/arkime_regression_test_harness/docker-compose.yml +++ b/arkime/arkime_regression_test_harness/docker-compose.yml @@ -7,7 +7,7 @@ services: environment: logger.level : 'INFO' bootstrap.memory_lock : 'true' - OPENSEARCH_JAVA_OPTS : '-Xms4g -Xmx4g -Xss256k -Djava.security.egd=file:/dev/./urandom' + OPENSEARCH_JAVA_OPTS : '-Xmx4g -Xms4g -Xss256k -Djava.security.egd=file:/dev/./urandom' discovery.type : 'single-node' cluster.routing.allocation.disk.threshold_enabled : 'false' cluster.routing.allocation.node_initial_primaries_recoveries : 8 diff --git a/config/logstash.env.example b/config/logstash.env.example index a9436ad51..e619a6375 100644 --- a/config/logstash.env.example +++ b/config/logstash.env.example @@ -21,4 +21,4 @@ LOGSTASH_NETBOX_CACHE_TTL=30 # Zeek log types that will be ignored (dropped) by LogStash LOGSTASH_ZEEK_IGNORED_LOGS=analyzer,broker,bsap_ip_unknown,bsap_serial_unknown,capture_loss,cluster,config,ecat_arp_info,loaded_scripts,packet_filter,png,print,prof,reporter,stats,stderr,stdout # Logstash memory allowance and other Java options -LS_JAVA_OPTS=-server -Xms2500m -Xmx2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true \ No newline at end of file +LS_JAVA_OPTS=-server -Xmx2500m -Xms2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true \ No newline at end of file diff --git a/config/opensearch.env.example b/config/opensearch.env.example index 95408c7f4..820854a40 100644 --- a/config/opensearch.env.example +++ b/config/opensearch.env.example @@ -39,7 +39,7 @@ OPENSEARCH_SECONDARY_CREDS_CONFIG_FILE=/var/local/curlrc/.opensearch.secondary.c # certificates). OPENSEARCH_SECONDARY_SSL_CERTIFICATE_VERIFICATION=false # OpenSearch memory allowance and other Java options -OPENSEARCH_JAVA_OPTS=-server -Xms10g -Xmx10g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true +OPENSEARCH_JAVA_OPTS=-server -Xmx10g -Xms10g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true # OpenSearch index patterns and timestamp fields # Index pattern for network traffic logs written via Logstash (e.g., Zeek logs, Suricata alerts) diff --git a/docs/malcolm-config.md b/docs/malcolm-config.md index 9632d6aba..a1d1bb6f7 100644 --- a/docs/malcolm-config.md +++ b/docs/malcolm-config.md @@ -38,7 +38,7 @@ Although the configuration script automates many of the following configuration - `LOGSTASH_SEVERITY_SCORING` - if set to `true`, Logstash will perform [severity scoring](severity.md#Severity) when analyzing Zeek logs (default `true`) - `LOGSTASH_NETBOX_ENRICHMENT` - if set to `true`, Logstash will [enrich network traffic metadata](asset-interaction-analysis.md#NetBoxEnrichment) via NetBox API calls - `LOGSTASH_NETBOX_AUTO_POPULATE` - if set to `true`, Logstash will [populate the NetBox inventory](asset-interaction-analysis.md#NetBoxPopPassive) based on observed network traffic - - `LS_JAVA_OPTS` - part of LogStash's [JVM settings](https://www.elastic.co/guide/en/logstash/current/jvm-settings.html), the `-Xms` and `-Xmx` values set the size of LogStash's Java heap (we recommend somewhere between `1500m` and `4g`) + - `LS_JAVA_OPTS` - part of LogStash's [JVM settings](https://www.elastic.co/guide/en/logstash/current/jvm-settings.html), the `-Xmx` and `-Xms` values set the size of LogStash's Java heap (we recommend somewhere between `1500m` and `4g`) * `pipeline.workers`, `pipeline.batch.size` and `pipeline.batch.delay` - these settings are used to tune the performance and resource utilization of the the `logstash` container; see [Tuning and Profiling Logstash Performance](https://www.elastic.co/guide/en/logstash/current/tuning-logstash.html), [`logstash.yml`](https://www.elastic.co/guide/en/logstash/current/logstash-settings-file.html) and [Multiple Pipelines](https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html) * **`lookup-common.env`** - settings for enrichment lookups, including those used for [customizing event severity scoring](severity.md#SeverityConfig) - `CONNECTION_SECONDS_SEVERITY_THRESHOLD` - when [severity scoring](severity.md#Severity) is enabled, this variable indicates the duration threshold (in seconds) for assigning severity to long connections (default `3600`) @@ -56,7 +56,7 @@ Although the configuration script automates many of the following configuration - `NGINX_LOG_ACCESS_AND_ERRORS` - if set to `true`, all access to Malcolm via its [web interfaces](quickstart.md#UserInterfaceURLs) will be logged to OpenSearch (default `false`) - `NGINX_SSL` - if set to `true`, require HTTPS connections to Malcolm's `nginx-proxy` container (default); if set to `false`, use unencrypted HTTP connections (using unsecured HTTP connections is **NOT** recommended unless you are running Malcolm behind another reverse proxy such as Traefik, Caddy, etc.) * **`opensearch.env`** - settings specific to [OpenSearch](https://opensearch.org/) - - `OPENSEARCH_JAVA_OPTS` - one of OpenSearch's most [important settings](https://opensearch.org/docs/latest/install-and-configure/install-opensearch/index/#important-settings), the `-Xms` and `-Xmx` values set the size of OpenSearch's Java heap (we recommend setting this value to half of system RAM, up to 32 gigabytes) + - `OPENSEARCH_JAVA_OPTS` - one of OpenSearch's most [important settings](https://opensearch.org/docs/latest/install-and-configure/install-opensearch/index/#important-settings), the `-Xmx` and `-Xms` values set the size of OpenSearch's Java heap (we recommend setting this value to half of system RAM, up to 32 gigabytes) - `OPENSEARCH_PRIMARY` - one of `opensearch-local`, `opensearch-remote`, or `elasticsearch-remote`, to determine the [OpenSearch or Elasticsearch instance](opensearch-instances.md#OpenSearchInstance) Malcolm will use (default `opensearch-local`) - `OPENSEARCH_URL` - when using Malcolm's internal OpenSearch instance (i.e., `OPENSEARCH_PRIMARY` is `opensearch-local`) this should be `http://opensearch:9200`, otherwise this value specifies the primary remote instance URL in the format `protocol://host:port` (default `http://opensearch:9200`) - `OPENSEARCH_SSL_CERTIFICATE_VERIFICATION` - if set to `true`, connections to the primary remote OpenSearch instance will require full TLS certificate validation (this may fail if using self-signed certificates) (default `false`) diff --git a/scripts/install.py b/scripts/install.py index 5e01049b9..18ad5a85a 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -89,8 +89,8 @@ MAC_BREW_DOCKER_COMPOSE_PACKAGE = 'docker-compose' MAC_BREW_DOCKER_SETTINGS = '/Users/{}/Library/Group Containers/group.com.docker/settings.json' -LOGSTASH_JAVA_OPTS_DEFAULT = '-server -Xms2500m -Xmx2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true' -OPENSEARCH_JAVA_OPTS_DEFAULT = '-server -Xms10g -Xmx10g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true' +LOGSTASH_JAVA_OPTS_DEFAULT = '-server -Xmx2500m -Xms2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true' +OPENSEARCH_JAVA_OPTS_DEFAULT = '-server -Xmx10g -Xms10g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true' ################################################################################################### ScriptName = os.path.basename(__file__) From 1e57a3a092f5a287f51ba395754963e27ca024a4 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 22 Feb 2024 11:40:53 -0700 Subject: [PATCH 30/79] for supporting JSON logs from Zeek (idaholab/Malcolm#65); getting closer --- logstash/pipelines/zeek/11_zeek_parse.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index 15926805c..5818835ab 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -1755,7 +1755,7 @@ filter { if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_rename_zeek_json_known_certs_fields" + id => "mutate_rename_zeek_json_known_hosts_fields" rename => { "[zeek_cols][host]" => "[zeek_cols][orig_h]" } } From def966194380e3bc9dada0626433a6edf80bb2b5 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 22 Feb 2024 13:13:19 -0700 Subject: [PATCH 31/79] Revert "bump logstash and beats to v8.12.2" This reverts commit 862d3d58cbb61dae803a0f514dadbc838f6d674c. --- Dockerfiles/filebeat.Dockerfile | 2 +- Dockerfiles/logstash.Dockerfile | 2 +- hedgehog-iso/build.sh | 2 +- hedgehog-raspi/sensor_install.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index 3c9502800..65b75e4e1 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/beats/filebeat-oss:8.12.2 +FROM docker.elastic.co/beats/filebeat-oss:8.12.1 # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/Dockerfiles/logstash.Dockerfile b/Dockerfiles/logstash.Dockerfile index 5505fb201..9e49f9deb 100644 --- a/Dockerfiles/logstash.Dockerfile +++ b/Dockerfiles/logstash.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/logstash/logstash-oss:8.12.2 +FROM docker.elastic.co/logstash/logstash-oss:8.12.1 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' diff --git a/hedgehog-iso/build.sh b/hedgehog-iso/build.sh index 16f2f4f92..67c0dfe5e 100755 --- a/hedgehog-iso/build.sh +++ b/hedgehog-iso/build.sh @@ -5,7 +5,7 @@ IMAGE_PUBLISHER=idaholab IMAGE_VERSION=1.0.0 IMAGE_DISTRIBUTION=bookworm -BEATS_VER="8.12.2" +BEATS_VER="8.12.1" BEATS_OSS="-oss" BUILD_ERROR_CODE=1 diff --git a/hedgehog-raspi/sensor_install.sh b/hedgehog-raspi/sensor_install.sh index 9ddd4f34f..d15225bcd 100644 --- a/hedgehog-raspi/sensor_install.sh +++ b/hedgehog-raspi/sensor_install.sh @@ -32,7 +32,7 @@ SHARED_DIR='/opt/buildshared' WORK_DIR="$(mktemp -d -t hedgehog-XXXXXX)" SENSOR_DIR='/opt/sensor' -BEATS_VER="8.12.2" +BEATS_VER="8.12.1" BEATS_OSS="-oss" # Option to build from sources if desired From d4094ca69c1a2635be45cd401c0264cc2f5b2200 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 26 Feb 2024 09:31:14 -0700 Subject: [PATCH 32/79] simplify extracted files download value actoin --- arkime/wise/source.zeeklogs.js | 2 +- dashboards/scripts/index-refresh.py | 2 +- nginx/nginx.conf | 9 +-------- nginx/nginx_readonly.conf | 9 +-------- 4 files changed, 4 insertions(+), 18 deletions(-) diff --git a/arkime/wise/source.zeeklogs.js b/arkime/wise/source.zeeklogs.js index 80fa72faa..68f791503 100644 --- a/arkime/wise/source.zeeklogs.js +++ b/arkime/wise/source.zeeklogs.js @@ -2219,7 +2219,7 @@ class MalcolmSource extends WISESource { // add right-click for extracted files from zeek var carvedFieldsStr = allFields.filter(value => /^zeek\.files\.extracted$/i.test(value)).join(','); - this.api.addValueAction("malcolm_carved_file_quarantined", { name: "Download", url: "/dl-extracted-files/%TEXT%", fields: carvedFieldsStr }); + this.api.addValueAction("malcolm_carved_file_quarantined", { name: "Download", url: "/extracted-files/%TEXT%", fields: carvedFieldsStr }); // add right-clicks for pivoting into dashboards from Arkime (see nginx.conf) var filterLabel = "OpenSearch Dashboards %DBFIELD%"; diff --git a/dashboards/scripts/index-refresh.py b/dashboards/scripts/index-refresh.py index 4e2ea7458..028b574af 100755 --- a/dashboards/scripts/index-refresh.py +++ b/dashboards/scripts/index-refresh.py @@ -416,7 +416,7 @@ def main(): elif re.search(r'(^zeek\.files\.extracted$)', field['name'], re.IGNORECASE) is not None: # add download for extracted zeek files drilldownInfoParamsUrlTemplateValues = {} - drilldownInfoParamsUrlTemplateValues['url'] = '/dl-extracted-files/{{value}}' + drilldownInfoParamsUrlTemplateValues['url'] = '/extracted-files/{{value}}' drilldownInfoParamsUrlTemplateValues['label'] = 'Download' drilldownInfoParamsUrlTemplates.append(drilldownInfoParamsUrlTemplateValues) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 87884cd6e..f05786b5c 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -178,14 +178,6 @@ http { proxy_set_header Authorization ""; } - # Dashboards/Arkime -> extracted file download - location ~* /dl-extracted-files/(.*) { - rewrite ^.*/dl-extracted-files/(.*) /extracted-files/$1 redirect; - proxy_pass http://extracted-file-http-server; - proxy_redirect off; - proxy_set_header Host file-monitor.malcolm.local; - } - # OpenSearch dashboards (or Kibana) location /dashboards { include /etc/nginx/nginx_dashboards_rewrite_rt.conf; @@ -199,6 +191,7 @@ http { proxy_set_header Host dashboards-helper.malcolm.local; } + # extracted file download location ~* ^/extracted-files\b(.*) { include /etc/nginx/nginx_auth_rt.conf; proxy_pass http://extracted-file-http-server$1; diff --git a/nginx/nginx_readonly.conf b/nginx/nginx_readonly.conf index 11c6b001e..1251cf46f 100644 --- a/nginx/nginx_readonly.conf +++ b/nginx/nginx_readonly.conf @@ -110,14 +110,6 @@ http { proxy_set_header Authorization ""; } - # Dashboards/Arkime -> extracted file download - location ~* /dl-extracted-files/(.*) { - rewrite ^.*/dl-extracted-files/(.*) /extracted-files/$1 redirect; - proxy_pass http://extracted-file-http-server; - proxy_redirect off; - proxy_set_header Host file-monitor.malcolm.local; - } - # OpenSearch dashboards (or Kibana) location /dashboards { include /etc/nginx/nginx_dashboards_rewrite_rt.conf; @@ -130,6 +122,7 @@ http { proxy_set_header Host dashboards-helper.malcolm.local; } + # extracted file download location ~* ^/extracted-files\b(.*) { proxy_pass http://extracted-file-http-server$1; proxy_redirect off; From b7aba3428a496561430004eb361cd08a148e5a91 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 26 Feb 2024 11:04:01 -0700 Subject: [PATCH 33/79] temporarily use testing branch for htadmin while I test a fix --- Dockerfiles/htadmin.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfiles/htadmin.Dockerfile b/Dockerfiles/htadmin.Dockerfile index 82718df06..ec22ca742 100644 --- a/Dockerfiles/htadmin.Dockerfile +++ b/Dockerfiles/htadmin.Dockerfile @@ -31,7 +31,7 @@ ENV PHP_VERSION $PHP_VERSION ENV MCRYPT_VERSION $MCRYPT_VERSION ENV BOOTSTRAP_VERSION $BOOTSTRAP_VERSION -ENV HTADMIN_URL "https://codeload.github.com/mmguero-dev/htadmin/tar.gz/master" +ENV HTADMIN_URL "https://codeload.github.com/mmguero-dev/htadmin/tar.gz/newlinefix" RUN apt-get -q update && \ apt-get -y -q --no-install-recommends upgrade && \ From cebdae7589e2b1fb0f300d63fbd81f5982fdd988 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 26 Feb 2024 14:17:20 -0700 Subject: [PATCH 34/79] restore branch for htadmin --- Dockerfiles/htadmin.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfiles/htadmin.Dockerfile b/Dockerfiles/htadmin.Dockerfile index ec22ca742..82718df06 100644 --- a/Dockerfiles/htadmin.Dockerfile +++ b/Dockerfiles/htadmin.Dockerfile @@ -31,7 +31,7 @@ ENV PHP_VERSION $PHP_VERSION ENV MCRYPT_VERSION $MCRYPT_VERSION ENV BOOTSTRAP_VERSION $BOOTSTRAP_VERSION -ENV HTADMIN_URL "https://codeload.github.com/mmguero-dev/htadmin/tar.gz/newlinefix" +ENV HTADMIN_URL "https://codeload.github.com/mmguero-dev/htadmin/tar.gz/master" RUN apt-get -q update && \ apt-get -y -q --no-install-recommends upgrade && \ From ec3c7f2be0de1af4ee5302aee30f3e4b344e318c Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 26 Feb 2024 14:42:47 -0700 Subject: [PATCH 35/79] fix idaholab/Malcolm#431, unencrypted, unzipped extracted file download not working --- .../scripts/extracted_files_http_server.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/file-monitor/scripts/extracted_files_http_server.py b/file-monitor/scripts/extracted_files_http_server.py index 3eaab24ff..e1f2e0743 100755 --- a/file-monitor/scripts/extracted_files_http_server.py +++ b/file-monitor/scripts/extracted_files_http_server.py @@ -344,15 +344,14 @@ def do_GET(self): ): # serve the asset file satisfied = True - ctype = self.guess_type(fullpath) - with open(fullpath, 'rb') as fhandle: - fs = os.fstat(fhandle.fileno()) + with open(fullpath, 'rb') as f: + fs = os.fstat(f.fileno()) self.send_response(200) self.send_header('Content-type', self.guess_type(fullpath)) self.send_header("Content-Length", str(fs[6])) self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) self.end_headers() - while chunk := fhandle.read(1024): + while chunk := f.read(1024): self.wfile.write(chunk) # handle regular file downloads @@ -402,7 +401,15 @@ def do_GET(self): else: # original file, unencrypted - SimpleHTTPRequestHandler.do_GET(self) + with open(fullpath, 'rb') as f: + fs = os.fstat(f.fileno()) + self.send_response(200) + self.send_header('Content-type', self.guess_type(fullpath)) + self.send_header("Content-Length", str(fs[6])) + self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) + self.end_headers() + while chunk := f.read(1024): + self.wfile.write(chunk) else: self.send_error(404, "Not Found") From 505cc91f92836d06239513e298dec1500a98078e Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 26 Feb 2024 15:32:54 -0700 Subject: [PATCH 36/79] work in progress incorporating icsnpp-profinet-io-cm, idaholab/Malcolm#429 --- arkime/etc/config.ini | 40 ++++++++++++++++ arkime/wise/source.zeeklogs.js | 36 +++++++++++++++ config/zeek.env.example | 1 + .../composable/component/zeek_ot.json | 36 +++++++++++++++ .../usr/local/etc/zeek/local.zeek | 4 ++ .../interface/sensor_ctl/control_vars.conf | 1 + logstash/maps/zeek_log_ecs_categories.yaml | 1 + logstash/pipelines/zeek/11_zeek_parse.conf | 46 +++++++++++++++++++ .../pipelines/zeek/13_zeek_normalize.conf | 15 ++++++ scripts/zeek_script_to_malcolm_boilerplate.py | 34 +++++++------- shared/bin/zeek_install_plugins.sh | 1 + zeek/config/local.zeek | 4 ++ 12 files changed, 203 insertions(+), 16 deletions(-) diff --git a/arkime/etc/config.ini b/arkime/etc/config.ini index ef02d8239..7f3ff2290 100644 --- a/arkime/etc/config.ini +++ b/arkime/etc/config.ini @@ -1485,6 +1485,45 @@ zeek.profinet_dce_rpc.activity_uuid=db:zeek.profinet_dce_rpc.activity_uuid;group zeek.profinet_dce_rpc.server_boot_time=db:zeek.profinet_dce_rpc.server_boot_time;group:zeek_profinet;kind:integer;viewerOnly:true;friendly:Server Boot Time;help:Server Boot Time zeek.profinet_dce_rpc.operation=db:zeek.profinet_dce_rpc.operation;group:zeek_profinet;kind:termfield;viewerOnly:true;friendly:Operation;help:Operation +# profinet_io_cm.log +# https://github.com/cisagov/icsnpp-profinet-io-cm +zeek.profinet_io_cm.rpc_version=db:zeek.profinet_io_cm.rpc_version;group:zeek_profinet_io_cm;kind:integer;friendly:RPC version;help:RPC version +zeek.profinet_io_cm.packet_type=db:zeek.profinet_io_cm.packet_type;group:zeek_profinet_io_cm;kind:termfield;friendly:Packet type;help:Packet type +zeek.profinet_io_cm.reserved_for_impl_1=db:zeek.profinet_io_cm.reserved_for_impl_1;group:zeek_profinet_io_cm;kind:termfield;friendly:Implementation flag 0;help:Implementation flag 0 +zeek.profinet_io_cm.last_fragment=db:zeek.profinet_io_cm.last_fragment;group:zeek_profinet_io_cm;kind:termfield;friendly:Last fragment flag;help:Last fragment flag +zeek.profinet_io_cm.fragment=db:zeek.profinet_io_cm.fragment;group:zeek_profinet_io_cm;kind:termfield;friendly:Fragment flag;help:Fragment flag +zeek.profinet_io_cm.no_fragment_requested=db:zeek.profinet_io_cm.no_fragment_requested;group:zeek_profinet_io_cm;kind:termfield;friendly:No fragment ACK req flag;help:No fragment ACK req flag +zeek.profinet_io_cm.maybe=db:zeek.profinet_io_cm.maybe;group:zeek_profinet_io_cm;kind:termfield;friendly:Maybe flag;help:Maybe flag +zeek.profinet_io_cm.idempotent=db:zeek.profinet_io_cm.idempotent;group:zeek_profinet_io_cm;kind:termfield;friendly:Idempotent flag;help:Idempotent flag +zeek.profinet_io_cm.broadcast=db:zeek.profinet_io_cm.broadcast;group:zeek_profinet_io_cm;kind:termfield;friendly:Broadcast flag;help:Broadcast flag +zeek.profinet_io_cm.reserved_for_impl_2=db:zeek.profinet_io_cm.reserved_for_impl_2;group:zeek_profinet_io_cm;kind:termfield;friendly:Implementation flag 7;help:Implementation flag 7 +zeek.profinet_io_cm.cancel_was_pending_at_call_end=db:zeek.profinet_io_cm.cancel_was_pending_at_call_end;group:zeek_profinet_io_cm;kind:termfield;friendly:Cancel pending;help:Cancel pending +zeek.profinet_io_cm.integer_encoding=db:zeek.profinet_io_cm.integer_encoding;group:zeek_profinet_io_cm;kind:termfield;friendly:Integer encoding;help:Integer encoding +zeek.profinet_io_cm.character_encoding=db:zeek.profinet_io_cm.character_encoding;group:zeek_profinet_io_cm;kind:termfield;friendly:Character encoding;help:Character encoding +zeek.profinet_io_cm.floating_point_encoding=db:zeek.profinet_io_cm.floating_point_encoding;group:zeek_profinet_io_cm;kind:termfield;friendly:Floating point encoding;help:Floating point encoding +zeek.profinet_io_cm.serial_high=db:zeek.profinet_io_cm.serial_high;group:zeek_profinet_io_cm;kind:integer;friendly:Fragment number high octet;help:Fragment number high octet +zeek.profinet_io_cm.object_uuid=db:zeek.profinet_io_cm.object_uuid;group:zeek_profinet_io_cm;kind:termfield;friendly:Object UUID;help:Object UUID +zeek.profinet_io_cm.interface_uuid=db:zeek.profinet_io_cm.interface_uuid;group:zeek_profinet_io_cm;kind:termfield;friendly:Interface UUID;help:Interface UUID +zeek.profinet_io_cm.activity_uuid=db:zeek.profinet_io_cm.activity_uuid;group:zeek_profinet_io_cm;kind:termfield;friendly:Activity UUID;help:Activity UUID +zeek.profinet_io_cm.server_boot_time=db:zeek.profinet_io_cm.server_boot_time;group:zeek_profinet_io_cm;kind:integer;friendly:Server boot time;help:Server boot time +zeek.profinet_io_cm.interface_vers_major=db:zeek.profinet_io_cm.interface_vers_major;group:zeek_profinet_io_cm;kind:integer;friendly:Interface version major;help:Interface version major +zeek.profinet_io_cm.interface_vers_minor=db:zeek.profinet_io_cm.interface_vers_minor;group:zeek_profinet_io_cm;kind:integer;friendly:Interface version minor;help:Interface version minor +zeek.profinet_io_cm.sequence_num=db:zeek.profinet_io_cm.sequence_num;group:zeek_profinet_io_cm;kind:integer;friendly:Activity sequence number;help:Activity sequence number +zeek.profinet_io_cm.operation=db:zeek.profinet_io_cm.operation;group:zeek_profinet_io_cm;kind:termfield;friendly:PNIO operation;help:PNIO operation +zeek.profinet_io_cm.interface_hint=db:zeek.profinet_io_cm.interface_hint;group:zeek_profinet_io_cm;kind:integer;friendly:Interface hint;help:Interface hint +zeek.profinet_io_cm.activity_hint=db:zeek.profinet_io_cm.activity_hint;group:zeek_profinet_io_cm;kind:integer;friendly:Activity hint;help:Activity hint +zeek.profinet_io_cm.len_of_body=db:zeek.profinet_io_cm.len_of_body;group:zeek_profinet_io_cm;kind:integer;friendly:NDRD data length;help:NDRD data length +zeek.profinet_io_cm.fragment_num=db:zeek.profinet_io_cm.fragment_num;group:zeek_profinet_io_cm;kind:integer;friendly:Fragment number;help:Fragment number +zeek.profinet_io_cm.auth_protocol=db:zeek.profinet_io_cm.auth_protocol;group:zeek_profinet_io_cm;kind:integer;friendly:Auth protocol;help:Auth protocol +zeek.profinet_io_cm.serial_low=db:zeek.profinet_io_cm.serial_low;group:zeek_profinet_io_cm;kind:integer;friendly:Fragment number low octet;help:Fragment number low octet +zeek.profinet_io_cm.vers_fack=db:zeek.profinet_io_cm.vers_fack;group:zeek_profinet_io_cm;kind:termfield;friendly:Version FACK;help:Version FACK +zeek.profinet_io_cm.window_size=db:zeek.profinet_io_cm.window_size;group:zeek_profinet_io_cm;kind:integer;friendly:Window size;help:Window size +zeek.profinet_io_cm.max_tsdu=db:zeek.profinet_io_cm.max_tsdu;group:zeek_profinet_io_cm;kind:integer;friendly:Maximum TSDU;help:Maximum TSDU +zeek.profinet_io_cm.max_frag_size=db:zeek.profinet_io_cm.max_frag_size;group:zeek_profinet_io_cm;kind:integer;friendly:Maximum fragment size;help:Maximum fragment size +zeek.profinet_io_cm.serial_number=db:zeek.profinet_io_cm.serial_number;group:zeek_profinet_io_cm;kind:integer;friendly:Serial number;help:Serial number +zeek.profinet_io_cm.sel_ack_len=db:zeek.profinet_io_cm.sel_ack_len;group:zeek_profinet_io_cm;kind:integer;friendly:Selective ACKs length;help:Selective ACKs length +zeek.profinet_io_cm.sel_ack=db:zeek.profinet_io_cm.sel_ack;group:zeek_profinet_io_cm;kind:termfield;friendly:Selective ACKs;help:Selective ACKs + # radius.log # https://docs.zeek.org/en/stable/scripts/base/protocols/radius/main.zeek.html#type-RADIUS::Info zeek.radius.mac=db:zeek.radius.mac;group:zeek_radius;kind:termfield;viewerOnly:true;friendly:MAC Address;help:MAC Address @@ -2653,6 +2692,7 @@ o_zeek_ospf=require:zeek.ospf;title:Zeek ospf.log;fields:zeek.ospf.ospf_type,zee o_zeek_pe=require:zeek.pe;title:Zeek pe.log;fields:zeek.pe.machine,zeek.pe.compile_ts,zeek.pe.os,zeek.pe.subsystem,zeek.pe.is_exe,zeek.pe.is_64bit,zeek.pe.uses_aslr,zeek.pe.uses_dep,zeek.pe.uses_code_integrity,zeek.pe.uses_seh,zeek.pe.has_import_table,zeek.pe.has_export_table,zeek.pe.has_cert_table,zeek.pe.has_debug_data,zeek.pe.section_names o_zeek_profinet=require:zeek.profinet;title:Zeek profinet.log;fields:zeek.profinet.operation_type,zeek.profinet.block_version,zeek.profinet.slot_number,zeek.profinet.subslot_number,zeek.profinet.index o_zeek_profinet_dce_rpc=require:zeek.profinet_dce_rpc;title:Zeek profinet_dce_rpc.log;fields:zeek.profinet_dce_rpc.version,zeek.profinet_dce_rpc.packet_type,zeek.profinet_dce_rpc.object_uuid,zeek.profinet_dce_rpc.interface_uuid,zeek.profinet_dce_rpc.activity_uuid,zeek.profinet_dce_rpc.server_boot_time,zeek.profinet_dce_rpc.operation +o_zeek_profinet_io_cm=require:zeek.profinet_io_cm;title:Zeek profinet_io_cm.log;fields:zeek.profinet_io_cm.rpc_version,zeek.profinet_io_cm.packet_type,zeek.profinet_io_cm.reserved_for_impl_1,zeek.profinet_io_cm.last_fragment,zeek.profinet_io_cm.fragment,zeek.profinet_io_cm.no_fragment_requested,zeek.profinet_io_cm.maybe,zeek.profinet_io_cm.idempotent,zeek.profinet_io_cm.broadcast,zeek.profinet_io_cm.reserved_for_impl_2,zeek.profinet_io_cm.cancel_was_pending_at_call_end,zeek.profinet_io_cm.integer_encoding,zeek.profinet_io_cm.character_encoding,zeek.profinet_io_cm.floating_point_encoding,zeek.profinet_io_cm.serial_high,zeek.profinet_io_cm.object_uuid,zeek.profinet_io_cm.interface_uuid,zeek.profinet_io_cm.activity_uuid,zeek.profinet_io_cm.server_boot_time,zeek.profinet_io_cm.interface_vers_major,zeek.profinet_io_cm.interface_vers_minor,zeek.profinet_io_cm.sequence_num,zeek.profinet_io_cm.operation,zeek.profinet_io_cm.interface_hint,zeek.profinet_io_cm.activity_hint,zeek.profinet_io_cm.len_of_body,zeek.profinet_io_cm.fragment_num,zeek.profinet_io_cm.auth_protocol,zeek.profinet_io_cm.serial_low,zeek.profinet_io_cm.vers_fack,zeek.profinet_io_cm.window_size,zeek.profinet_io_cm.max_tsdu,zeek.profinet_io_cm.max_frag_size,zeek.profinet_io_cm.serial_number,zeek.profinet_io_cm.sel_ack_len,zeek.profinet_io_cm.sel_ack o_zeek_radius=require:zeek.radius;title:Zeek radius.log;fields:zeek.radius.mac,zeek.radius.framed_addr,zeek.radius.tunnel_client,zeek.radius.connect_info,zeek.radius.reply_msg,zeek.radius.result,zeek.radius.ttl o_zeek_rdp=require:zeek.rdp;title:Zeek rdp.log;fields:zeek.rdp.cookie,zeek.rdp.result,zeek.rdp.security_protocol,zeek.rdp.client_channels,zeek.rdp.keyboard_layout,zeek.rdp.client_build,zeek.rdp.client_name,zeek.rdp.client_dig_product_id,zeek.rdp.desktop_width,zeek.rdp.desktop_height,zeek.rdp.requested_color_depth,zeek.rdp.cert_type,zeek.rdp.cert_count,zeek.rdp.cert_permanent,zeek.rdp.encryption_level,zeek.rdp.encryption_method o_zeek_rfb=require:zeek.rfb;title:Zeek rfb.log;fields:zeek.rfb.client_major_version,zeek.rfb.client_minor_version,zeek.rfb.server_major_version,zeek.rfb.server_minor_version,zeek.rfb.authentication_method,zeek.rfb.auth,zeek.rfb.share_flag,zeek.rfb.desktop_name,zeek.rfb.width,zeek.rfb.height diff --git a/arkime/wise/source.zeeklogs.js b/arkime/wise/source.zeeklogs.js index 68f791503..1a47c019e 100644 --- a/arkime/wise/source.zeeklogs.js +++ b/arkime/wise/source.zeeklogs.js @@ -1719,6 +1719,42 @@ class MalcolmSource extends WISESource { "zeek.profinet_dce_rpc.packet_type", "zeek.profinet_dce_rpc.server_boot_time", "zeek.profinet_dce_rpc.version", + "zeek.profinet_io_cm.rpc_version", + "zeek.profinet_io_cm.packet_type", + "zeek.profinet_io_cm.reserved_for_impl_1", + "zeek.profinet_io_cm.last_fragment", + "zeek.profinet_io_cm.fragment", + "zeek.profinet_io_cm.no_fragment_requested", + "zeek.profinet_io_cm.maybe", + "zeek.profinet_io_cm.idempotent", + "zeek.profinet_io_cm.broadcast", + "zeek.profinet_io_cm.reserved_for_impl_2", + "zeek.profinet_io_cm.cancel_was_pending_at_call_end", + "zeek.profinet_io_cm.integer_encoding", + "zeek.profinet_io_cm.character_encoding", + "zeek.profinet_io_cm.floating_point_encoding", + "zeek.profinet_io_cm.serial_high", + "zeek.profinet_io_cm.object_uuid", + "zeek.profinet_io_cm.interface_uuid", + "zeek.profinet_io_cm.activity_uuid", + "zeek.profinet_io_cm.server_boot_time", + "zeek.profinet_io_cm.interface_vers_major", + "zeek.profinet_io_cm.interface_vers_minor", + "zeek.profinet_io_cm.sequence_num", + "zeek.profinet_io_cm.operation_num", + "zeek.profinet_io_cm.interface_hint", + "zeek.profinet_io_cm.activity_hint", + "zeek.profinet_io_cm.len_of_body", + "zeek.profinet_io_cm.fragment_num", + "zeek.profinet_io_cm.auth_protocol", + "zeek.profinet_io_cm.serial_low", + "zeek.profinet_io_cm.vers_fack", + "zeek.profinet_io_cm.window_size", + "zeek.profinet_io_cm.max_tsdu", + "zeek.profinet_io_cm.max_frag_size", + "zeek.profinet_io_cm.serial_number", + "zeek.profinet_io_cm.sel_ack_len", + "zeek.profinet_io_cm.sel_ack", "zeek.radius.connect_info", "zeek.radius.framed_addr", "zeek.radius.mac", diff --git a/config/zeek.env.example b/config/zeek.env.example index 880e583e3..a0bd28d98 100644 --- a/config/zeek.env.example +++ b/config/zeek.env.example @@ -84,6 +84,7 @@ ZEEK_DISABLE_ICS_GENISYS=true ZEEK_DISABLE_ICS_OPCUA_BINARY= ZEEK_DISABLE_ICS_MODBUS= ZEEK_DISABLE_ICS_PROFINET= +ZEEK_DISABLE_ICS_PROFINET_IO_CM= ZEEK_DISABLE_ICS_S7COMM= ZEEK_DISABLE_ICS_SYNCHROPHASOR= ZEEK_SYNCHROPHASOR_PORTS= diff --git a/dashboards/templates/composable/component/zeek_ot.json b/dashboards/templates/composable/component/zeek_ot.json index 2ed2174ef..55917246f 100644 --- a/dashboards/templates/composable/component/zeek_ot.json +++ b/dashboards/templates/composable/component/zeek_ot.json @@ -691,6 +691,42 @@ "zeek.profinet_dce_rpc.packet_type": { "type": "keyword" }, "zeek.profinet_dce_rpc.server_boot_time": { "type": "integer" }, "zeek.profinet_dce_rpc.version": { "type": "integer" }, + "zeek.profinet_io_cm.rpc_version": { "type": "long" }, + "zeek.profinet_io_cm.packet_type": { "type": "keyword" }, + "zeek.profinet_io_cm.reserved_for_impl_1": { "type": "keyword" }, + "zeek.profinet_io_cm.last_fragment": { "type": "keyword" }, + "zeek.profinet_io_cm.fragment": { "type": "keyword" }, + "zeek.profinet_io_cm.no_fragment_requested": { "type": "keyword" }, + "zeek.profinet_io_cm.maybe": { "type": "keyword" }, + "zeek.profinet_io_cm.idempotent": { "type": "keyword" }, + "zeek.profinet_io_cm.broadcast": { "type": "keyword" }, + "zeek.profinet_io_cm.reserved_for_impl_2": { "type": "keyword" }, + "zeek.profinet_io_cm.cancel_was_pending_at_call_end": { "type": "keyword" }, + "zeek.profinet_io_cm.integer_encoding": { "type": "keyword" }, + "zeek.profinet_io_cm.character_encoding": { "type": "keyword" }, + "zeek.profinet_io_cm.floating_point_encoding": { "type": "keyword" }, + "zeek.profinet_io_cm.serial_high": { "type": "long" }, + "zeek.profinet_io_cm.object_uuid": { "type": "keyword" }, + "zeek.profinet_io_cm.interface_uuid": { "type": "keyword" }, + "zeek.profinet_io_cm.activity_uuid": { "type": "keyword" }, + "zeek.profinet_io_cm.server_boot_time": { "type": "long" }, + "zeek.profinet_io_cm.interface_vers_major": { "type": "long" }, + "zeek.profinet_io_cm.interface_vers_minor": { "type": "long" }, + "zeek.profinet_io_cm.sequence_num": { "type": "long" }, + "zeek.profinet_io_cm.operation": { "type": "keyword" }, + "zeek.profinet_io_cm.interface_hint": { "type": "long" }, + "zeek.profinet_io_cm.activity_hint": { "type": "long" }, + "zeek.profinet_io_cm.len_of_body": { "type": "long" }, + "zeek.profinet_io_cm.fragment_num": { "type": "long" }, + "zeek.profinet_io_cm.auth_protocol": { "type": "long" }, + "zeek.profinet_io_cm.serial_low": { "type": "long" }, + "zeek.profinet_io_cm.vers_fack": { "type": "keyword" }, + "zeek.profinet_io_cm.window_size": { "type": "long" }, + "zeek.profinet_io_cm.max_tsdu": { "type": "long" }, + "zeek.profinet_io_cm.max_frag_size": { "type": "long" }, + "zeek.profinet_io_cm.serial_number": { "type": "long" }, + "zeek.profinet_io_cm.sel_ack_len": { "type": "long" }, + "zeek.profinet_io_cm.sel_ack": { "type": "keyword" }, "zeek.s7comm.error_class": { "type": "keyword" }, "zeek.s7comm.error_code": { "type": "keyword" }, "zeek.s7comm.function_code": { "type": "keyword" }, diff --git a/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek index 0d27093e6..1827f0449 100644 --- a/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek +++ b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek @@ -40,6 +40,7 @@ global disable_ics_genisys = (getenv("ZEEK_DISABLE_ICS_GENISYS") == true_regex) global disable_ics_opcua_binary = (getenv("ZEEK_DISABLE_ICS_OPCUA_BINARY") == true_regex) ? T : F; global disable_ics_modbus = (getenv("ZEEK_DISABLE_ICS_MODBUS") == true_regex) ? T : F; global disable_ics_profinet = (getenv("ZEEK_DISABLE_ICS_PROFINET") == true_regex) ? T : F; +global disable_ics_profinet_io_cm = (getenv("ZEEK_DISABLE_ICS_PROFINET_IO_CM") == true_regex) ? T : F; global disable_ics_s7comm = (getenv("ZEEK_DISABLE_ICS_S7COMM") == true_regex) ? T : F; global disable_ics_synchrophasor = (getenv("ZEEK_DISABLE_ICS_SYNCHROPHASOR") == true_regex) ? T : F; @@ -146,6 +147,9 @@ event zeek_init() &priority=-5 { if (disable_ics_all || disable_ics_profinet) { Analyzer::disable_analyzer(Analyzer::ANALYZER_PROFINET); } + if (disable_ics_all || disable_ics_profinet_io_cm) { + Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SPICY_PROFINET_IO_CM); + } if (disable_ics_all || disable_ics_s7comm) { Analyzer::disable_analyzer(Analyzer::ANALYZER_S7COMM_TCP); } diff --git a/hedgehog-iso/interface/sensor_ctl/control_vars.conf b/hedgehog-iso/interface/sensor_ctl/control_vars.conf index 16998d5c4..56d4b8985 100644 --- a/hedgehog-iso/interface/sensor_ctl/control_vars.conf +++ b/hedgehog-iso/interface/sensor_ctl/control_vars.conf @@ -82,6 +82,7 @@ export ZEEK_DISABLE_ICS_GENISYS=true export ZEEK_DISABLE_ICS_OPCUA_BINARY= export ZEEK_DISABLE_ICS_MODBUS= export ZEEK_DISABLE_ICS_PROFINET= +export ZEEK_DISABLE_ICS_PROFINET_IO_CM= export ZEEK_DISABLE_ICS_S7COMM= export ZEEK_DISABLE_ICS_SYNCHROPHASOR= export ZEEK_SYNCHROPHASOR_PORTS= diff --git a/logstash/maps/zeek_log_ecs_categories.yaml b/logstash/maps/zeek_log_ecs_categories.yaml index 5a27fc270..6f7e94d18 100644 --- a/logstash/maps/zeek_log_ecs_categories.yaml +++ b/logstash/maps/zeek_log_ecs_categories.yaml @@ -61,6 +61,7 @@ "pe": ["file"] "profinet": ["ot", "network"] "profinet_dce_rpc": ["ot", "network"] +"profinet_io_cm": ["ot", "network"] "radius": ["authentication", "iam", "network"] "rdp": ["network"] "rfb": ["network"] diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index 5818835ab..ff9e67ade 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -2671,6 +2671,52 @@ filter { add_tag => [ "ics" ] } + } else if ([log_source] == "profinet_io_cm") { + ############################################################################################################################# + # profinet_io_cm.log + # https://github.com/cisagov/icsnpp-profinet-io-cm + + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_profinet_io_cm_fields" + rename => { "[zeek_cols][array_of_sel_ack]" => "[zeek_cols][sel_ack]" } + rename => { "[zeek_cols][operation_num]" => "[zeek_cols][operation]" } + } + + } else { + dissect { + id => "dissect_zeek_profinet_io_cm" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][rpc_version]} %{[zeek_cols][packet_type]} %{[zeek_cols][reserved_for_impl_1]} %{[zeek_cols][last_fragment]} %{[zeek_cols][fragment]} %{[zeek_cols][no_fragment_requested]} %{[zeek_cols][maybe]} %{[zeek_cols][idempotent]} %{[zeek_cols][broadcast]} %{[zeek_cols][reserved_for_impl_2]} %{[zeek_cols][cancel_was_pending_at_call_end]} %{[zeek_cols][integer_encoding]} %{[zeek_cols][character_encoding]} %{[zeek_cols][floating_point_encoding]} %{[zeek_cols][serial_high]} %{[zeek_cols][object_uuid]} %{[zeek_cols][interface_uuid]} %{[zeek_cols][activity_uuid]} %{[zeek_cols][server_boot_time]} %{[zeek_cols][interface_vers_major]} %{[zeek_cols][interface_vers_minor]} %{[zeek_cols][sequence_num]} %{[zeek_cols][operation]} %{[zeek_cols][interface_hint]} %{[zeek_cols][activity_hint]} %{[zeek_cols][len_of_body]} %{[zeek_cols][fragment_num]} %{[zeek_cols][auth_protocol]} %{[zeek_cols][serial_low]} %{[zeek_cols][vers_fack]} %{[zeek_cols][window_size]} %{[zeek_cols][max_tsdu]} %{[zeek_cols][max_frag_size]} %{[zeek_cols][serial_number]} %{[zeek_cols][sel_ack_len]} %{[zeek_cols][sel_ack]}" + } + } + + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_profinet_io_cm" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_profinet_io_cm" + init => "$zeek_profinet_io_cm_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'rpc_version', 'packet_type', 'reserved_for_impl_1', 'last_fragment', 'fragment', 'no_fragment_requested', 'maybe', 'idempotent', 'broadcast', 'reserved_for_impl_2', 'cancel_was_pending_at_call_end', 'integer_encoding', 'character_encoding', 'floating_point_encoding', 'serial_high', 'object_uuid', 'interface_uuid', 'activity_uuid', 'server_boot_time', 'interface_vers_major', 'interface_vers_minor', 'sequence_num', 'operation', 'interface_hint', 'activity_hint', 'len_of_body', 'fragment_num', 'auth_protocol', 'serial_low', 'vers_fack', 'window_size', 'max_tsdu', 'max_frag_size', 'serial_number', 'sel_ack_len', 'sel_ack' ]" + code => "event.set('[zeek_cols]', $zeek_profinet_io_cm_field_names.zip(event.get('[message]')).to_h)" + } + } + mutate { id => "mutate_split_zeek_profinet_io_cm_commas" + split => { "[zeek_cols][sel_ack]" => "," } } + } + + mutate { + id => "mutate_add_fields_zeek_profinet_io_cm" + add_field => { + "[zeek_cols][proto]" => "udp" + "[zeek_cols][service]" => "profinet_io_cm" + } + add_tag => [ "ics" ] + } + } else if ([log_source] == "radius") { ############################################################################################################################# # radius.log diff --git a/logstash/pipelines/zeek/13_zeek_normalize.conf b/logstash/pipelines/zeek/13_zeek_normalize.conf index ddaf21874..9ce6218a9 100644 --- a/logstash/pipelines/zeek/13_zeek_normalize.conf +++ b/logstash/pipelines/zeek/13_zeek_normalize.conf @@ -115,6 +115,9 @@ filter { if ([zeek][profinet_dce_rpc][version]) { mutate { id => "mutate_merge_normalize_zeek_profinet_dce_rpc_version" merge => { "[network][protocol_version]" => "[zeek][profinet_dce_rpc][version]" } } } + if ([zeek][profinet_io_cm][rpc_version]) { mutate { id => "mutate_merge_normalize_zeek_profinet_io_cm_version" + merge => { "[network][protocol_version]" => "[zeek][profinet_io_cm][rpc_version]" } } } + if ([zeek][rfb]) { ruby { id => "ruby_zeek_field_network_protocol_version_rfb" @@ -436,6 +439,18 @@ filter { if ([zeek][profinet_dce_rpc][operation]) { mutate { id => "mutate_merge_normalize_zeek_profinet_dce_rpc_operation" merge => { "[event][action]" => "[zeek][profinet_dce_rpc][operation]" } } } + if ([zeek][profinet_io_cm]) { + ruby { + # action = packet_type:operation + id => "ruby_zeek_profinet_io_cm_generate_action" + code => " + actions = Array.new unless (actions = event.get('[event][action]')) + actions.append([event.get('[zeek][profinet_io_cm][packet_type]'), + event.get('[zeek][profinet_io_cm][operation]')].reject{ |e| e.nil? || e&.empty? }.join(':')) + event.set('[event][action]', actions)" + } + } + if ([zeek][rfb][auth]) and ([zeek][rfb][authentication_method]) { # if authentication was attempted, assign an "authenticate" action mutate { id => "mutate_add_field_zeek_rfb_auth_action" diff --git a/scripts/zeek_script_to_malcolm_boilerplate.py b/scripts/zeek_script_to_malcolm_boilerplate.py index f254099e0..d93323d77 100755 --- a/scripts/zeek_script_to_malcolm_boilerplate.py +++ b/scripts/zeek_script_to_malcolm_boilerplate.py @@ -487,24 +487,26 @@ def main(): f' # {rName}.log', f' # {os.path.basename(val)} ({args.url})', '', - f' dissect {{', - f' id => "dissect_zeek_{rName}"', - f" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP", - f' mapping => {{', - f' "[message]" => "{rFieldsDissect}"', - f' }}', - f' }}', - '', - f' if ("_dissectfailure" in [tags]) {{', - f' mutate {{', - f' id => "mutate_split_zeek_{rName}"', + f' if ("_jsonparsesuccess" not in [tags]) {{', + f' dissect {{', + f' id => "dissect_zeek_{rName}"', f" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP", - f' split => {{ "[message]" => "{ZEEK_DELIMITER_CHAR}" }}', + f' mapping => {{', + f' "[message]" => "{rFieldsDissect}"', + f' }}', f' }}', - f' ruby {{', - f' id => "ruby_zip_zeek_{rName}"', - f' init => "$zeek_{rName}_field_names = [ {rFieldsZip} ]"', - f" code => \"event.set('[zeek_cols]', $zeek_{rName}_field_names.zip(event.get('[message]')).to_h)\"", + '', + f' if ("_dissectfailure" in [tags]) {{', + f' mutate {{', + f' id => "mutate_split_zeek_{rName}"', + f" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP", + f' split => {{ "[message]" => "{ZEEK_DELIMITER_CHAR}" }}', + f' }}', + f' ruby {{', + f' id => "ruby_zip_zeek_{rName}"', + f' init => "$zeek_{rName}_field_names = [ {rFieldsZip} ]"', + f" code => \"event.set('[zeek_cols]', $zeek_{rName}_field_names.zip(event.get('[message]')).to_h)\"", + f' }}', f' }}', f' }}', '', diff --git a/shared/bin/zeek_install_plugins.sh b/shared/bin/zeek_install_plugins.sh index 70ce9b01f..e521a08e5 100755 --- a/shared/bin/zeek_install_plugins.sh +++ b/shared/bin/zeek_install_plugins.sh @@ -76,6 +76,7 @@ ZKG_GITHUB_URLS=( "https://github.com/cisagov/icsnpp-genisys" "https://github.com/cisagov/icsnpp-modbus" "https://github.com/cisagov/icsnpp-opcua-binary" + "https://github.com/cisagov/icsnpp-profinet-io-cm" "https://github.com/cisagov/icsnpp-s7comm" "https://github.com/cisagov/icsnpp-synchrophasor" "https://github.com/corelight/callstranger-detector" diff --git a/zeek/config/local.zeek b/zeek/config/local.zeek index 8f04cf0e8..868e3c082 100644 --- a/zeek/config/local.zeek +++ b/zeek/config/local.zeek @@ -40,6 +40,7 @@ global disable_ics_genisys = (getenv("ZEEK_DISABLE_ICS_GENISYS") == true_regex) global disable_ics_opcua_binary = (getenv("ZEEK_DISABLE_ICS_OPCUA_BINARY") == true_regex) ? T : F; global disable_ics_modbus = (getenv("ZEEK_DISABLE_ICS_MODBUS") == true_regex) ? T : F; global disable_ics_profinet = (getenv("ZEEK_DISABLE_ICS_PROFINET") == true_regex) ? T : F; +global disable_ics_profinet_io_cm = (getenv("ZEEK_DISABLE_ICS_PROFINET_IO_CM") == true_regex) ? T : F; global disable_ics_s7comm = (getenv("ZEEK_DISABLE_ICS_S7COMM") == true_regex) ? T : F; global disable_ics_synchrophasor = (getenv("ZEEK_DISABLE_ICS_SYNCHROPHASOR") == true_regex) ? T : F; @@ -146,6 +147,9 @@ event zeek_init() &priority=-5 { if (disable_ics_all || disable_ics_profinet) { Analyzer::disable_analyzer(Analyzer::ANALYZER_PROFINET); } + if (disable_ics_all || disable_ics_profinet_io_cm) { + Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SPICY_PROFINET_IO_CM); + } if (disable_ics_all || disable_ics_s7comm) { Analyzer::disable_analyzer(Analyzer::ANALYZER_S7COMM_TCP); } From d213a79938c37b28fb60b76e3cbb265c051e4093 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 26 Feb 2024 15:38:21 -0700 Subject: [PATCH 37/79] work in progress incorporating icsnpp-profinet-io-cm, idaholab/Malcolm#429 --- docs/protocols.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/protocols.md b/docs/protocols.md index e40e11084..abda373b3 100644 --- a/docs/protocols.md +++ b/docs/protocols.md @@ -33,6 +33,7 @@ Malcolm uses [Zeek](https://docs.zeek.org/en/stable/script-reference/proto-analy |OpenVPN|[🔗](https://en.wikipedia.org/wiki/OpenVPN)|[🔗](https://openvpn.net/community-resources/openvpn-protocol/)[🔗](https://zeek.org/2021/03/16/a-zeek-openvpn-protocol-analyzer/)||[✓](https://github.com/corelight/zeek-spicy-openvpn)| |PostgreSQL|[🔗](https://en.wikipedia.org/wiki/PostgreSQL)|[🔗](https://www.postgresql.org/)|[✓](https://github.com/arkime/arkime/blob/master/capture/parsers/postgresql.c)|| |Process Field Net (PROFINET)|[🔗](https://en.wikipedia.org/wiki/PROFINET)|[🔗](https://us.profinet.com/technology/profinet/)||[✓](https://github.com/amzn/zeek-plugin-profinet/blob/master/scripts/main.zeek)| +|PROFINET IO CM (Input/Output Context Manager)|[🔗](https://wiki.wireshark.org/PROFINET/IO)|[🔗](https://us.profinet.com/technology/profinet/)[🔗](https://webstore.iec.ch/publication/83418)||[✓](https://github.com/cisagov/icsnpp-profinet-io-cm/blob/main/analyzer/types.zeek)| |Remote Authentication Dial-In User Service (RADIUS)|[🔗](https://en.wikipedia.org/wiki/RADIUS)|[🔗](https://tools.ietf.org/html/rfc2865)|[✓](https://github.com/arkime/arkime/blob/master/capture/parsers/radius.c)|[✓](https://docs.zeek.org/en/stable/scripts/base/protocols/radius/main.zeek.html#type-RADIUS::Info)| |Remote Desktop Protocol (RDP)|[🔗](https://en.wikipedia.org/wiki/Remote_Desktop_Protocol)|[🔗](https://docs.microsoft.com/en-us/windows/win32/termserv/remote-desktop-protocol?redirectedfrom=MSDN)||[✓](https://docs.zeek.org/en/stable/scripts/base/protocols/rdp/main.zeek.html#type-RDP::Info)| |Remote Framebuffer (RFB)|[🔗](https://en.wikipedia.org/wiki/RFB_protocol)|[🔗](https://tools.ietf.org/html/rfc6143)||[✓](https://docs.zeek.org/en/stable/scripts/base/protocols/rfb/main.zeek.html#type-RFB::Info)| From b6c0c8aecc9b7e223339130890f6c35e4e0507bc Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 26 Feb 2024 15:44:45 -0700 Subject: [PATCH 38/79] work in progress incorporating icsnpp-profinet-io-cm, idaholab/Malcolm#429 --- Dockerfiles/zeek.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index d85956cd8..63f18408c 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -160,8 +160,8 @@ ADD shared/bin/zeekdeploy.sh ${ZEEK_DIR}/bin/ # sanity checks to make sure the plugins installed and copied over correctly # these ENVs should match the number of third party scripts/plugins installed by zeek_install_plugins.sh -ENV ZEEK_THIRD_PARTY_PLUGINS_COUNT 22 -ENV ZEEK_THIRD_PARTY_PLUGINS_GREP "(Zeek::Spicy|ANALYZER_SPICY_DHCP|ANALYZER_SPICY_DNS|ANALYZER_SPICY_HTTP|ANALYZER_SPICY_OSPF|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SYNCHROPHASOR_TCP|ANALYZER_GENISYS_TCP|ANALYZER_S7COMM_TCP|Corelight::CommunityID|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|ICSNPP::OPCUA_Binary|Salesforce::GQUIC|Zeek::PROFINET|Zeek::TDS)" +ENV ZEEK_THIRD_PARTY_PLUGINS_COUNT 23 +ENV ZEEK_THIRD_PARTY_PLUGINS_GREP "(Zeek::Spicy|ANALYZER_SPICY_DHCP|ANALYZER_SPICY_DNS|ANALYZER_SPICY_HTTP|ANALYZER_SPICY_OSPF|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SYNCHROPHASOR_TCP|ANALYZER_GENISYS_TCP|ANALYZER_SPICY_PROFINET_IO_CM|ANALYZER_S7COMM_TCP|Corelight::CommunityID|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|ICSNPP::OPCUA_Binary|Salesforce::GQUIC|Zeek::PROFINET|Zeek::TDS)" ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 25 ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP "(bro-is-darknet/main|bro-simple-scan/scan|bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|cve-2022-22954/main|cve-2022-26809/main|CVE-2022-3602/__load__|hassh/hassh|http-more-files-names/main|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)" From 1e019f4949344be87a0c5ba10b9af07584dcddf0 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 26 Feb 2024 16:08:40 -0700 Subject: [PATCH 39/79] work in progress incorporating icsnpp-profinet-io-cm, idaholab/Malcolm#429 --- logstash/pipelines/zeek/11_zeek_parse.conf | 1 - logstash/pipelines/zeek/13_zeek_normalize.conf | 16 +++++----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index ff9e67ade..e6d1ed795 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -2712,7 +2712,6 @@ filter { id => "mutate_add_fields_zeek_profinet_io_cm" add_field => { "[zeek_cols][proto]" => "udp" - "[zeek_cols][service]" => "profinet_io_cm" } add_tag => [ "ics" ] } diff --git a/logstash/pipelines/zeek/13_zeek_normalize.conf b/logstash/pipelines/zeek/13_zeek_normalize.conf index 9ce6218a9..8c0d62063 100644 --- a/logstash/pipelines/zeek/13_zeek_normalize.conf +++ b/logstash/pipelines/zeek/13_zeek_normalize.conf @@ -439,17 +439,11 @@ filter { if ([zeek][profinet_dce_rpc][operation]) { mutate { id => "mutate_merge_normalize_zeek_profinet_dce_rpc_operation" merge => { "[event][action]" => "[zeek][profinet_dce_rpc][operation]" } } } - if ([zeek][profinet_io_cm]) { - ruby { - # action = packet_type:operation - id => "ruby_zeek_profinet_io_cm_generate_action" - code => " - actions = Array.new unless (actions = event.get('[event][action]')) - actions.append([event.get('[zeek][profinet_io_cm][packet_type]'), - event.get('[zeek][profinet_io_cm][operation]')].reject{ |e| e.nil? || e&.empty? }.join(':')) - event.set('[event][action]', actions)" - } - } + if ([zeek][profinet_io_cm][packet_type]) { mutate { id => "mutate_merge_normalize_zeek_profinet_io_cm_packet_type" + merge => { "[event][action]" => "[zeek][profinet_io_cm][packet_type]" } } } + + if ([zeek][profinet_io_cm][operation]) { mutate { id => "mutate_merge_normalize_zeek_profinet_io_cm_operation" + merge => { "[event][action]" => "[zeek][profinet_io_cm][operation]" } } } if ([zeek][rfb][auth]) and ([zeek][rfb][authentication_method]) { # if authentication was attempted, assign an "authenticate" action From 713b4a0dfef5faf1e8124171a51ad4a63978e91a Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 27 Feb 2024 10:08:46 -0700 Subject: [PATCH 40/79] work in progress incorporating icsnpp-profinet-io-cm, idaholab/Malcolm#429 --- config/logstash.env.example | 2 +- .../usr/local/etc/zeek/local.zeek | 33 +++++++++++++++++++ logstash/pipelines/zeek/11_zeek_parse.conf | 19 +++++------ zeek/config/local.zeek | 33 +++++++++++++++++++ 4 files changed, 75 insertions(+), 12 deletions(-) diff --git a/config/logstash.env.example b/config/logstash.env.example index e619a6375..f734bf17e 100644 --- a/config/logstash.env.example +++ b/config/logstash.env.example @@ -19,6 +19,6 @@ LOGSTASH_NETBOX_AUTO_POPULATE=false LOGSTASH_NETBOX_CACHE_SIZE=1000 LOGSTASH_NETBOX_CACHE_TTL=30 # Zeek log types that will be ignored (dropped) by LogStash -LOGSTASH_ZEEK_IGNORED_LOGS=analyzer,broker,bsap_ip_unknown,bsap_serial_unknown,capture_loss,cluster,config,ecat_arp_info,loaded_scripts,packet_filter,png,print,prof,reporter,stats,stderr,stdout +LOGSTASH_ZEEK_IGNORED_LOGS=analyzer,broker,capture_loss,cluster,config,loaded_scripts,packet_filter,png,print,prof,reporter,stats,stderr,stdout # Logstash memory allowance and other Java options LS_JAVA_OPTS=-server -Xmx2500m -Xms2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true \ No newline at end of file diff --git a/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek index 1827f0449..c1c305f94 100644 --- a/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek +++ b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek @@ -301,3 +301,36 @@ redef CVE_2021_44228::log = F; break; } @endif + +##! Other logs we're just disabling unilaterally + +# amzn/zeek-plugin-profinet's profinet_dce_rpc.log is covered by cisagov/icsnpp-profinet-io-cm +hook Profinet::log_policy_profinet_dce_rpc( + rec: Profinet::Profinet_DCE_RPC, + id: Log::ID, + filter: Log::Filter) { + break; +} + +# we're not tracking the BSAP "unknown" logs +hook Bsap::log_policy_bsap_ip_unknown( + rec: Bsap::BSAP_IP_UNKNOWN, + id: Log::ID, + filter: Log::Filter) { + break; +} + +hook Bsap::log_policy_bsap_serial_unknown( + rec: Bsap::BSAP_SERIAL_UNKNOWN, + id: Log::ID, + filter: Log::Filter) { + break; +} + +# all ARP traffic is logged by the ethercat parser, which is overkill +hook PacketAnalyzer::ECAT::log_policy_ecat_arp( + rec: PacketAnalyzer::ECAT::ECAT_ARP_INFO, + id: Log::ID, + filter: Log::Filter) { + break; +} diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index e6d1ed795..aacee02f2 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -29,11 +29,13 @@ filter { # be assigning to them if we were reading TSV if ("_jsonparsesuccess" in [tags]) { + # some of the ICSNPP parsers do an interesting thing to handle source and destination fields + # (see https://github.com/cisagov/icsnpp-bacnet/?tab=readme-ov-file#source-and-destination-fields) + # so check for and handle those first ruby { id => "ruby_zeek_json_determine_source_destination_fields" code => " - if !event.get('[zeek_cols][is_orig]').nil? and - ![event.get('[zeek_cols][source_h]').to_s, + if ![event.get('[zeek_cols][source_h]').to_s, event.get('[zeek_cols][source_p]').to_s, event.get('[zeek_cols][destination_h]').to_s, event.get('[zeek_cols][destination_p]').to_s].reject{ |e| e.nil? || e.empty? || (e == '0') }.empty? then @@ -41,11 +43,7 @@ filter { end " } - if ([@metadata][icsnpp_source_dest_fields]) { - # some of the ICSNPP parsers do an interesting thing to handle source and destination fields - # (see https://github.com/cisagov/icsnpp-bacnet/?tab=readme-ov-file#source-and-destination-fields) - # so check for and handle those first mutate { id => "mutate_rename_zeek_json_common_reversed_direction_fields" rename => { "[zeek_cols][id.orig_h]" => "[zeek_cols][drop_orig_h]" } @@ -1247,9 +1245,8 @@ filter { # ecat_arp_info.log # https://github.com/cisagov/icsnpp-ethercat # - # NOTE: I currently have this disabled by including ecat_arp_info in LOGSTASH_ZEEK_IGNORED_LOGS - # to be handled bdrop_zeek_ignored_source above, as it is including ALL arps and not just those - # from ethercat traffic which can be misleading (i.e., indicating ecat traffic where there is none) + # NOTE: I currently have this disabled via policy hook in local.zeek, as it is including ALL arps and + # not just those from ethercat traffic which can be misleading (i.e., indicating ecat traffic where there is none) if ("_jsonparsesuccess" in [tags]) { mutate { @@ -2688,7 +2685,7 @@ filter { id => "dissect_zeek_profinet_io_cm" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][rpc_version]} %{[zeek_cols][packet_type]} %{[zeek_cols][reserved_for_impl_1]} %{[zeek_cols][last_fragment]} %{[zeek_cols][fragment]} %{[zeek_cols][no_fragment_requested]} %{[zeek_cols][maybe]} %{[zeek_cols][idempotent]} %{[zeek_cols][broadcast]} %{[zeek_cols][reserved_for_impl_2]} %{[zeek_cols][cancel_was_pending_at_call_end]} %{[zeek_cols][integer_encoding]} %{[zeek_cols][character_encoding]} %{[zeek_cols][floating_point_encoding]} %{[zeek_cols][serial_high]} %{[zeek_cols][object_uuid]} %{[zeek_cols][interface_uuid]} %{[zeek_cols][activity_uuid]} %{[zeek_cols][server_boot_time]} %{[zeek_cols][interface_vers_major]} %{[zeek_cols][interface_vers_minor]} %{[zeek_cols][sequence_num]} %{[zeek_cols][operation]} %{[zeek_cols][interface_hint]} %{[zeek_cols][activity_hint]} %{[zeek_cols][len_of_body]} %{[zeek_cols][fragment_num]} %{[zeek_cols][auth_protocol]} %{[zeek_cols][serial_low]} %{[zeek_cols][vers_fack]} %{[zeek_cols][window_size]} %{[zeek_cols][max_tsdu]} %{[zeek_cols][max_frag_size]} %{[zeek_cols][serial_number]} %{[zeek_cols][sel_ack_len]} %{[zeek_cols][sel_ack]}" + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][rpc_version]} %{[zeek_cols][packet_type]} %{[zeek_cols][reserved_for_impl_1]} %{[zeek_cols][last_fragment]} %{[zeek_cols][fragment]} %{[zeek_cols][no_fragment_requested]} %{[zeek_cols][maybe]} %{[zeek_cols][idempotent]} %{[zeek_cols][broadcast]} %{[zeek_cols][reserved_for_impl_2]} %{[zeek_cols][cancel_was_pending_at_call_end]} %{[zeek_cols][integer_encoding]} %{[zeek_cols][character_encoding]} %{[zeek_cols][floating_point_encoding]} %{[zeek_cols][serial_high]} %{[zeek_cols][object_uuid]} %{[zeek_cols][interface_uuid]} %{[zeek_cols][activity_uuid]} %{[zeek_cols][server_boot_time]} %{[zeek_cols][interface_vers_major]} %{[zeek_cols][interface_vers_minor]} %{[zeek_cols][sequence_num]} %{[zeek_cols][operation]} %{[zeek_cols][interface_hint]} %{[zeek_cols][activity_hint]} %{[zeek_cols][len_of_body]} %{[zeek_cols][fragment_num]} %{[zeek_cols][auth_protocol]} %{[zeek_cols][serial_low]} %{[zeek_cols][vers_fack]} %{[zeek_cols][window_size]} %{[zeek_cols][max_tsdu]} %{[zeek_cols][max_frag_size]} %{[zeek_cols][serial_number]} %{[zeek_cols][sel_ack_len]} %{[zeek_cols][sel_ack]}" } } @@ -2700,7 +2697,7 @@ filter { } ruby { id => "ruby_zip_zeek_profinet_io_cm" - init => "$zeek_profinet_io_cm_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'rpc_version', 'packet_type', 'reserved_for_impl_1', 'last_fragment', 'fragment', 'no_fragment_requested', 'maybe', 'idempotent', 'broadcast', 'reserved_for_impl_2', 'cancel_was_pending_at_call_end', 'integer_encoding', 'character_encoding', 'floating_point_encoding', 'serial_high', 'object_uuid', 'interface_uuid', 'activity_uuid', 'server_boot_time', 'interface_vers_major', 'interface_vers_minor', 'sequence_num', 'operation', 'interface_hint', 'activity_hint', 'len_of_body', 'fragment_num', 'auth_protocol', 'serial_low', 'vers_fack', 'window_size', 'max_tsdu', 'max_frag_size', 'serial_number', 'sel_ack_len', 'sel_ack' ]" + init => "$zeek_profinet_io_cm_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'rpc_version', 'packet_type', 'reserved_for_impl_1', 'last_fragment', 'fragment', 'no_fragment_requested', 'maybe', 'idempotent', 'broadcast', 'reserved_for_impl_2', 'cancel_was_pending_at_call_end', 'integer_encoding', 'character_encoding', 'floating_point_encoding', 'serial_high', 'object_uuid', 'interface_uuid', 'activity_uuid', 'server_boot_time', 'interface_vers_major', 'interface_vers_minor', 'sequence_num', 'operation', 'interface_hint', 'activity_hint', 'len_of_body', 'fragment_num', 'auth_protocol', 'serial_low', 'vers_fack', 'window_size', 'max_tsdu', 'max_frag_size', 'serial_number', 'sel_ack_len', 'sel_ack' ]" code => "event.set('[zeek_cols]', $zeek_profinet_io_cm_field_names.zip(event.get('[message]')).to_h)" } } diff --git a/zeek/config/local.zeek b/zeek/config/local.zeek index 868e3c082..521c2c087 100644 --- a/zeek/config/local.zeek +++ b/zeek/config/local.zeek @@ -301,3 +301,36 @@ redef CVE_2021_44228::log = F; break; } @endif + +##! Other logs we're just disabling unilaterally + +# amzn/zeek-plugin-profinet's profinet_dce_rpc.log is covered by cisagov/icsnpp-profinet-io-cm +hook Profinet::log_policy_profinet_dce_rpc( + rec: Profinet::Profinet_DCE_RPC, + id: Log::ID, + filter: Log::Filter) { + break; +} + +# we're not tracking the BSAP "unknown" logs +hook Bsap::log_policy_bsap_ip_unknown( + rec: Bsap::BSAP_IP_UNKNOWN, + id: Log::ID, + filter: Log::Filter) { + break; +} + +hook Bsap::log_policy_bsap_serial_unknown( + rec: Bsap::BSAP_SERIAL_UNKNOWN, + id: Log::ID, + filter: Log::Filter) { + break; +} + +# all ARP traffic is logged by the ethercat parser, which is overkill +hook PacketAnalyzer::ECAT::log_policy_ecat_arp( + rec: PacketAnalyzer::ECAT::ECAT_ARP_INFO, + id: Log::ID, + filter: Log::Filter) { + break; +} From 5e6c33be6fcece5edca533cd6aca52469fdb653a Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 27 Feb 2024 10:46:01 -0700 Subject: [PATCH 41/79] work in progress incorporating icsnpp-profinet-io-cm, idaholab/Malcolm#429 --- logstash/pipelines/zeek/11_zeek_parse.conf | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index aacee02f2..034c7e569 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -107,18 +107,22 @@ filter { # normalize service string(s) + # For some reason, even in JSON, I have service strings like: + # ...,"proto":"udp","service":"profinet,profinet_dce_rpc,spicy_profinet_io_cm,profinet",... + # so whatever reason it's not already an array. Split it here. + mutate { id => "mutate_split_zeek_conn_commas" + split => { "[zeek_cols][service]" => "," } } + # some services are named like blah_udp/blah_tcp/blah_data, and we don't care about the suffix mutate { id => "mutate_gsub_field_zeek_conn_service_protocol_suffix" gsub => [ "[zeek_cols][service]", "[_-](tcp|udp|data)", "" ] } - if ([zeek_cols][service] =~ /spicy_/) { - # if it's coming from spicy, we don't care to have that in the service name - mutate { id => "mutate_gsub_field_zeek_conn_service_spicy_prefix" - gsub => [ "[zeek_cols][service]", "spicy_", "" ] } + # if it's coming from spicy, we don't care to have that in the service name + mutate { id => "mutate_gsub_field_zeek_conn_service_spicy_prefix" + gsub => [ "[zeek_cols][service]", "spicy_", "" ] } - mutate { id => "mutate_gsub_field_zeek_conn_service_spicy_cipher_suffix" - gsub => [ "[zeek_cols][service]", "(_hmac)?(_(sha|md)\d+)?$", "" ] } - } + mutate { id => "mutate_gsub_field_zeek_conn_service_spicy_cipher_suffix" + gsub => [ "[zeek_cols][service]", "(_hmac)?(_(sha|md)\d+)?$", "" ] } if ([zeek_cols][orig_ip_bytes]) and ([zeek_cols][orig_ip_bytes] != '-') and ([zeek_cols][orig_ip_bytes] != '(empty)') and ([zeek_cols][orig_ip_bytes] != '') { mutate { id => "mutate_add_field_zeek_srcBytes" From d1718b174d85975a18253b1377858b1c5a2c8e20 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 27 Feb 2024 11:17:12 -0700 Subject: [PATCH 42/79] work in progress incorporating icsnpp-profinet-io-cm, idaholab/Malcolm#429 --- .../a7514350-eba6-11e9-a384-0fcf32210194.json | 228 ++++++++---------- 1 file changed, 100 insertions(+), 128 deletions(-) diff --git a/dashboards/dashboards/a7514350-eba6-11e9-a384-0fcf32210194.json b/dashboards/dashboards/a7514350-eba6-11e9-a384-0fcf32210194.json index cafab2e32..c51cbcd3b 100644 --- a/dashboards/dashboards/a7514350-eba6-11e9-a384-0fcf32210194.json +++ b/dashboards/dashboards/a7514350-eba6-11e9-a384-0fcf32210194.json @@ -1,5 +1,5 @@ { - "version": "7.10.0", + "version": "2.12.0", "objects": [ { "id": "a7514350-eba6-11e9-a384-0fcf32210194", @@ -7,18 +7,18 @@ "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY2MSwxXQ==", + "updated_at": "2024-02-27T18:15:37.621Z", + "version": "Wzk5MywxXQ==", "attributes": { "title": "PROFINET", "hits": 0, "description": "", - "panelsJSON": "[{\"version\":\"7.6.2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":8,\"h\":69,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"7.6.2\",\"gridData\":{\"x\":8,\"y\":0,\"w\":8,\"h\":14,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"7.6.2\",\"gridData\":{\"x\":16,\"y\":0,\"w\":32,\"h\":14,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"7.6.2\",\"gridData\":{\"x\":0,\"y\":69,\"w\":48,\"h\":34,\"i\":\"4\"},\"panelIndex\":\"4\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"7.6.2\",\"gridData\":{\"x\":8,\"y\":50,\"w\":19,\"h\":19,\"i\":\"5\"},\"panelIndex\":\"5\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"},{\"version\":\"7.6.2\",\"gridData\":{\"x\":27,\"y\":50,\"w\":21,\"h\":19,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"},{\"version\":\"7.6.2\",\"gridData\":{\"x\":8,\"y\":14,\"w\":16,\"h\":19,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"},{\"version\":\"7.6.2\",\"gridData\":{\"x\":24,\"y\":14,\"w\":24,\"h\":19,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{},\"panelRefName\":\"panel_7\"},{\"version\":\"7.6.2\",\"gridData\":{\"x\":8,\"y\":33,\"w\":19,\"h\":17,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{},\"panelRefName\":\"panel_8\"},{\"version\":\"7.6.2\",\"gridData\":{\"x\":27,\"y\":33,\"w\":21,\"h\":17,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{},\"panelRefName\":\"panel_9\"}]", + "panelsJSON": "[{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":8,\"h\":33,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":8,\"y\":0,\"w\":8,\"h\":14,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":16,\"y\":0,\"w\":32,\"h\":14,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":33,\"w\":12,\"h\":34,\"i\":\"5\"},\"panelIndex\":\"5\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":2,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_3\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":12,\"y\":33,\"w\":12,\"h\":34,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":2,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_4\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":8,\"y\":14,\"w\":16,\"h\":19,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":24,\"y\":14,\"w\":24,\"h\":19,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":24,\"y\":33,\"w\":24,\"h\":34,\"i\":\"f4ece1af-6321-41fb-91fd-187f98ab9098\"},\"panelIndex\":\"f4ece1af-6321-41fb-91fd-187f98ab9098\",\"embeddableConfig\":{},\"panelRefName\":\"panel_7\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":67,\"w\":48,\"h\":31,\"i\":\"76674c9f-4ea3-40cb-8c76-a8aa4d77677a\"},\"panelIndex\":\"76674c9f-4ea3-40cb-8c76-a8aa4d77677a\",\"embeddableConfig\":{},\"panelRefName\":\"panel_8\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":98,\"w\":48,\"h\":28,\"i\":\"5827d746-068a-41b6-afa8-4c93ad914dae\"},\"panelIndex\":\"5827d746-068a-41b6-afa8-4c93ad914dae\",\"embeddableConfig\":{},\"panelRefName\":\"panel_9\"}]", "optionsJSON": "{\"useMargins\":true}", "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[],\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}}}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\",\"time_zone\":\"America/Denver\"}}},\"filter\":[]}" } }, "references": [ @@ -39,38 +39,38 @@ }, { "name": "panel_3", - "type": "search", - "id": "a0a10870-eba5-11e9-a384-0fcf32210194" + "type": "visualization", + "id": "ec42baa0-eba8-11e9-a384-0fcf32210194" }, { "name": "panel_4", "type": "visualization", - "id": "ec42baa0-eba8-11e9-a384-0fcf32210194" + "id": "0957f330-eba9-11e9-a384-0fcf32210194" }, { "name": "panel_5", "type": "visualization", - "id": "0957f330-eba9-11e9-a384-0fcf32210194" + "id": "41f36a70-ebaa-11e9-a384-0fcf32210194" }, { "name": "panel_6", "type": "visualization", - "id": "41f36a70-ebaa-11e9-a384-0fcf32210194" + "id": "9dccb5f0-eba9-11e9-a384-0fcf32210194" }, { "name": "panel_7", "type": "visualization", - "id": "9dccb5f0-eba9-11e9-a384-0fcf32210194" + "id": "dde62a00-d59b-11ee-becf-dfae5e789f77" }, { "name": "panel_8", - "type": "visualization", - "id": "17319090-ebab-11e9-a384-0fcf32210194" + "type": "search", + "id": "a0a10870-eba5-11e9-a384-0fcf32210194" }, { "name": "panel_9", - "type": "visualization", - "id": "8022cc90-ebab-11e9-a384-0fcf32210194" + "type": "search", + "id": "fde4f550-d598-11ee-becf-dfae5e789f77" } ], "migrationVersion": { @@ -83,8 +83,8 @@ "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:25:09.616Z", - "version": "Wzg3MiwxXQ==", + "updated_at": "2024-02-27T17:29:26.729Z", + "version": "Wzg2MiwxXQ==", "attributes": { "title": "Network Logs", "visState": "{\"title\":\"Network Logs\",\"type\":\"markdown\",\"params\":{\"markdown\":\"### General\\n[Overview](#/dashboard/0ad3d7c2-3441-485e-9dfe-dbb22e84e576) \\n[Security Overview](#/dashboard/95479950-41f2-11ea-88fa-7151df485405) \\n[ICS/IoT Security Overview](#/dashboard/4a4bde20-4760-11ea-949c-bbb5a9feecbf) \\n[Severity](#/dashboard/d2dd0180-06b1-11ec-8c6b-353266ade330) \\n[Connections](#/dashboard/abdd7550-2c7c-40dc-947e-f6d186a158c4) \\n[Actions and Results](#/dashboard/a33e0a50-afcd-11ea-993f-b7d8522a8bed) \\n[Files](#/dashboard/9ee51f94-3316-4fc5-bd89-93a52af69714) \\n[Executables](#/dashboard/0a490422-0ce9-44bf-9a2d-19329ddde8c3) \\n[Software](#/dashboard/87d990cc-9e0b-41e5-b8fe-b10ae1da0c85) \\n[Zeek Known Summary](#/dashboard/89d1cc50-974c-11ed-bb6b-3fb06c879b11) \\n[Zeek Intelligence](#/dashboard/36ed695f-edcc-47c1-b0ec-50d20c93ce0f) \\n[Zeek Notices](#/dashboard/f1f09567-fc7f-450b-a341-19d2f2bb468b) \\n[Zeek Weird](#/dashboard/1fff49f6-0199-4a0f-820b-721aff9ff1f1) \\n[Signatures](#/dashboard/665d1610-523d-11e9-a30e-e3576242f3ed) \\n[Suricata Alerts](#/dashboard/5694ca60-cbdf-11ec-a50a-5fedd672f5c5) \\n[Asset Interaction Analysis](#/dashboard/677ee170-809e-11ed-8d5b-07069f823b6f) \\n[↪ NetBox](/netbox/) \\n[↪ Arkime](/arkime/) \\n\\n### Common Protocols\\n[DCE/RPC](#/dashboard/432af556-c5c0-4cc3-8166-b274b4e3a406) ● [DHCP](#/dashboard/2d98bb8e-214c-4374-837b-20e1bcd63a5e) ● [DNS](#/dashboard/2cf94cd0-ecab-40a5-95a7-8419f3a39cd9) ● [FTP](#/dashboard/078b9aa5-9bd4-4f02-ae5e-cf80fa6f887b) / [TFTP](#/dashboard/bf5efbb0-60f1-11eb-9d60-dbf0411cfc48) ● [HTTP](#/dashboard/37041ee1-79c0-4684-a436-3173b0e89876) ● [IRC](#/dashboard/76f2f912-80da-44cd-ab66-6a73c8344cc3) ● [Kerberos](#/dashboard/82da3101-2a9c-4ae2-bb61-d447a3fbe673) ● [LDAP](#/dashboard/05e3e000-f118-11e9-acda-83a8e29e1a24) ● [MQTT](#/dashboard/87a32f90-ef58-11e9-974e-9d600036d105) ● [MySQL](#/dashboard/50ced171-1b10-4c3f-8b67-2db9635661a6) ● [NTLM](#/dashboard/543118a9-02d7-43fe-b669-b8652177fc37) ● [NTP](#/dashboard/af5df620-eeb6-11e9-bdef-65a192b7f586) ● [OSPF](#/dashboard/1cc01ff0-5205-11ec-a62c-7bc80e88f3f0) ● [QUIC](#/dashboard/11ddd980-e388-11e9-b568-cf17de8e860c) ● [RADIUS](#/dashboard/ae79b7d1-4281-4095-b2f6-fa7eafda9970) ● [RDP](#/dashboard/7f41913f-cba8-43f5-82a8-241b7ead03e0) ● [RFB](#/dashboard/f77bf097-18a8-465c-b634-eb2acc7a4f26) ● [SIP](#/dashboard/0b2354ae-0fe9-4fd9-b156-1c3870e5c7aa) ● [SMB](#/dashboard/42e831b9-41a9-4f35-8b7d-e1566d368773) ● [SMTP](#/dashboard/bb827f8e-639e-468c-93c8-9f5bc132eb8f) ● [SNMP](#/dashboard/4e5f106e-c60a-4226-8f64-d534abb912ab) ● [SSH](#/dashboard/caef3ade-d289-4d05-a511-149f3e97f238) ● [SSL](#/dashboard/7f77b58a-df3e-4cc2-b782-fd7f8bad8ffb) / [X.509 Certificates](#/dashboard/024062a6-48d6-498f-a91a-3bf2da3a3cd3) ● [STUN](#/dashboard/fa477130-2b8a-11ec-a9f2-3911c8571bfd) ● [Syslog](#/dashboard/92985909-dc29-4533-9e80-d3182a0ecf1d) ● [TDS](#/dashboard/bed185a0-ef82-11e9-b38a-2db3ee640e88) / [TDS RPC](#/dashboard/32587740-ef88-11e9-b38a-2db3ee640e88) / [TDS SQL](#/dashboard/fa141950-ef89-11e9-b38a-2db3ee640e88) ● [Telnet / rlogin / rsh](#/dashboard/c2549e10-7f2e-11ea-9f8a-1fe1327e2cd2) ● [Tunnels](#/dashboard/11be6381-beef-40a7-bdce-88c5398392fc)\\n\\n### ICS/IoT Protocols\\n[BACnet](#/dashboard/2bec1490-eb94-11e9-a384-0fcf32210194) ● [BSAP](#/dashboard/ca5799a0-56b5-11eb-b749-576de068f8ad) ● [DNP3](#/dashboard/870a5862-6c26-4a08-99fd-0c06cda85ba3) ● [EtherCAT](#/dashboard/4a073440-b286-11eb-a4d4-09fa12a6ebd4) ● [EtherNet/IP](#/dashboard/29a1b290-eb98-11e9-a384-0fcf32210194) ● [GENISYS](#/dashboard/03207c00-d07e-11ec-b4a7-d1b4003706b7) ● [Modbus](#/dashboard/152f29dc-51a2-4f53-93e9-6e92765567b8) ● [OPCUA Binary](#/dashboard/dd87edd0-796a-11ec-9ce6-b395c1ff58f4) ● [PROFINET](#/dashboard/a7514350-eba6-11e9-a384-0fcf32210194) ● [S7comm](#/dashboard/e76d05c0-eb9f-11e9-a384-0fcf32210194) ● [Synchrophasor](#/dashboard/2cc56240-e460-11ed-a9d5-9f591c284cb4) ● [Best Guess](#/dashboard/12e3a130-d83b-11eb-a0b0-f328ce09b0b7)\",\"type\":\"markdown\",\"fontSize\":10,\"openLinksInNewTab\":false},\"aggs\":[]}", @@ -106,28 +106,28 @@ "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY2MywxXQ==", + "updated_at": "2024-02-27T18:00:32.722Z", + "version": "Wzk2NiwxXQ==", "attributes": { "title": "PROFINET - Log Count", - "visState": "{\"title\":\"PROFINET - Log Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\",\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":36}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Log Count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"event.dataset\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Log Type\"}}]}", + "visState": "{\"title\":\"PROFINET - Log Count\",\"type\":\"metric\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{\"customLabel\":\"Log Count\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.dataset\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Log Type\"},\"schema\":\"group\"}],\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\",\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":36}}}}", "uiStateJSON": "{}", "description": "", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[{\"meta\":{\"type\":\"phrases\",\"key\":\"event.dataset\",\"value\":\"profinet, profinet_dce_rpc\",\"params\":[\"profinet\",\"profinet_dce_rpc\"],\"negate\":false,\"disabled\":false,\"alias\":\"Zeek Log Type\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"event.dataset\":\"profinet\"}},{\"match_phrase\":{\"event.dataset\":\"profinet_dce_rpc\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[{\"meta\":{\"type\":\"phrases\",\"key\":\"event.dataset\",\"value\":\"profinet, profinet_dce_rpc, profinet_io_cm\",\"params\":[\"profinet\",\"profinet_dce_rpc\",\"profinet_io_cm\"],\"alias\":\"Zeek Log Type\",\"negate\":false,\"disabled\":false,\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"event.dataset\":\"profinet\"}},{\"match_phrase\":{\"event.dataset\":\"profinet_dce_rpc\"}},{\"match_phrase\":{\"event.dataset\":\"profinet_io_cm\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ { "name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern", - "id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" + "id": "arkime_sessions3-*" }, { "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", "type": "index-pattern", - "id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" + "id": "arkime_sessions3-*" } ], "migrationVersion": { @@ -140,89 +140,45 @@ "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY2NCwxXQ==", + "updated_at": "2024-02-27T17:59:35.087Z", + "version": "Wzk2NSwxXQ==", "attributes": { "title": "PROFINET - Logs Over Time", - "visState": "{\"title\":\"PROFINET - Logs Over Time\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER\",\"timeRange\":{\"from\":\"now-25y\",\"to\":\"now\",\"mode\":\"relative\"},\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\" \"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"event.dataset\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Log Type\"}}]}", - "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", + "visState": "{\"title\":\"PROFINET - Logs Over Time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"firstPacket\",\"timeRange\":{\"from\":\"now-15y\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\" \"},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.dataset\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Log Type\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", "description": "", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[{\"meta\":{\"type\":\"phrases\",\"key\":\"event.dataset\",\"value\":\"profinet, profinet_dce_rpc\",\"params\":[\"profinet\",\"profinet_dce_rpc\"],\"negate\":false,\"disabled\":false,\"alias\":\"Zeek Log Type\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"event.dataset\":\"profinet\"}},{\"match_phrase\":{\"event.dataset\":\"profinet_dce_rpc\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[{\"meta\":{\"type\":\"phrases\",\"key\":\"event.dataset\",\"value\":\"profinet, profinet_dce_rpc, profinet_io_cm\",\"params\":[\"profinet\",\"profinet_dce_rpc\",\"profinet_io_cm\"],\"alias\":\"Zeek Log Type\",\"negate\":false,\"disabled\":false,\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"event.dataset\":\"profinet\"}},{\"match_phrase\":{\"event.dataset\":\"profinet_dce_rpc\"}},{\"match_phrase\":{\"event.dataset\":\"profinet_io_cm\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ { "name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern", - "id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" + "id": "arkime_sessions3-*" }, { "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", "type": "index-pattern", - "id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" + "id": "arkime_sessions3-*" } ], "migrationVersion": { "visualization": "7.10.0" } }, - { - "id": "a0a10870-eba5-11e9-a384-0fcf32210194", - "type": "search", - "namespaces": [ - "default" - ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY2NSwxXQ==", - "attributes": { - "title": "PROFINET and Related - Logs", - "description": "", - "hits": 0, - "columns": [ - "source.ip", - "source.port", - "destination.ip", - "destination.port", - "zeek.profinet.operation_type", - "zeek.profinet.index", - "zeek.profinet_dce_rpc.operation", - "event.id" - ], - "sort": [ - [ - "MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER", - "desc" - ] - ], - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"event.dataset:(profinet OR profinet_dce_rpc)\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" - } - }, - "references": [ - { - "name": "kibanaSavedObjectMeta.searchSourceJSON.index", - "type": "index-pattern", - "id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" - } - ], - "migrationVersion": { - "search": "7.9.3" - } - }, { "id": "ec42baa0-eba8-11e9-a384-0fcf32210194", "type": "visualization", "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY2NiwxXQ==", + "updated_at": "2024-02-27T18:13:58.684Z", + "version": "Wzk5MSwxXQ==", "attributes": { "title": "PROFINET - Source IP", - "visState": "{\"title\":\"PROFINET - Source IP\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source.ip\",\"size\":250,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source IP\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source.port\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source Port\"}}]}", + "visState": "{\"title\":\"PROFINET - Source IP\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source.ip\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":250,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source IP\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source.port\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source Port\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":20,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}}}}", "description": "", "version": 1, @@ -233,8 +189,8 @@ }, "references": [ { - "type": "search", "name": "search_0", + "type": "search", "id": "a0a10870-eba5-11e9-a384-0fcf32210194" } ], @@ -248,11 +204,11 @@ "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY2NywxXQ==", + "updated_at": "2024-02-27T18:14:38.259Z", + "version": "Wzk5MiwxXQ==", "attributes": { "title": "PROFINET - Destination IP", - "visState": "{\"title\":\"PROFINET - Destination IP\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"destination.ip\",\"size\":250,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Destination IP\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"destination.port\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Destination Port\"}}]}", + "visState": "{\"title\":\"PROFINET - Destination IP\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.ip\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":250,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Destination IP\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.port\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Destination Port\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":20,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}}}}", "description": "", "version": 1, @@ -263,8 +219,8 @@ }, "references": [ { - "type": "search", "name": "search_0", + "type": "search", "id": "a0a10870-eba5-11e9-a384-0fcf32210194" } ], @@ -278,8 +234,8 @@ "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY2OCwxXQ==", + "updated_at": "2024-02-27T17:29:03.431Z", + "version": "WzYxMCwxXQ==", "attributes": { "title": "PROFINET - Operation", "visState": "{\"title\":\"PROFINET - Operation\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":4,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.profinet.operation_type\",\"size\":250,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Operation\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.profinet.index\",\"size\":30,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Index\"}}]}", @@ -308,8 +264,8 @@ "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY2OSwxXQ==", + "updated_at": "2024-02-27T17:29:03.431Z", + "version": "WzYxMSwxXQ==", "attributes": { "title": "PROFINET - Operation Details", "visState": "{\"title\":\"PROFINET - Operation Details\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":4,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.profinet.operation_type\",\"size\":250,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Operation\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.profinet.index\",\"size\":30,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Index\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.profinet.slot_number\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Slot\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.profinet.subslot_number\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Subslot\"}}]}", @@ -333,29 +289,29 @@ } }, { - "id": "17319090-ebab-11e9-a384-0fcf32210194", + "id": "dde62a00-d59b-11ee-becf-dfae5e789f77", "type": "visualization", "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY3MCwxXQ==", + "updated_at": "2024-02-27T18:13:11.455Z", + "version": "Wzk5MCwxXQ==", "attributes": { - "title": "PROFINET DCE/RPC - Operation", - "visState": "{\"title\":\"PROFINET DCE/RPC - Operation\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"zeek.profinet_dce_rpc.operation\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Operation\"}}]}", - "uiStateJSON": "{}", + "title": "PROFINET I/O CM - Operation", + "visState": "{\"title\":\"PROFINET I/O CM - Operation\",\"type\":\"horizontal_bar\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.profinet_io_cm.operation\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Operation\"},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.profinet_io_cm.packet_type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Other\",\"customLabel\":\"Packet Type\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":200},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":75,\"filter\":true,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":false,\"labels\":{},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", "description": "", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" }, "savedSearchRefName": "search_0" }, "references": [ { - "type": "search", "name": "search_0", - "id": "96d31d60-eba5-11e9-a384-0fcf32210194" + "type": "search", + "id": "fde4f550-d598-11ee-becf-dfae5e789f77" } ], "migrationVersion": { @@ -363,45 +319,60 @@ } }, { - "id": "8022cc90-ebab-11e9-a384-0fcf32210194", - "type": "visualization", + "id": "a0a10870-eba5-11e9-a384-0fcf32210194", + "type": "search", "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY3MSwxXQ==", + "updated_at": "2024-02-27T18:05:08.977Z", + "version": "Wzk4OCwxXQ==", "attributes": { - "title": "PROFINET DCE/RPC - Packet Type", - "visState": "{\"title\":\"PROFINET DCE/RPC - Packet Type\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"zeek.profinet_dce_rpc.packet_type\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Packet Type\"}}]}", - "uiStateJSON": "{}", + "title": "PROFINET and Related - Logs", "description": "", + "hits": 0, + "columns": [ + "event.dataset", + "source.ip", + "source.port", + "destination.ip", + "destination.port", + "network.protocol_version", + "event.action", + "zeek.profinet.index", + "event.id" + ], + "sort": [ + [ + "firstPacket", + "desc" + ] + ], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" - }, - "savedSearchRefName": "search_0" + "searchSourceJSON": "{\"query\":{\"query\":\"event.dataset:(profinet OR profinet_dce_rpc OR profinet_io_cm)\",\"language\":\"lucene\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"firstPacket\",\"fixed_interval\":\"30d\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } }, "references": [ { - "type": "search", - "name": "search_0", - "id": "96d31d60-eba5-11e9-a384-0fcf32210194" + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "arkime_sessions3-*" } ], "migrationVersion": { - "visualization": "7.10.0" + "search": "7.9.3" } }, { - "id": "8524e670-eba5-11e9-a384-0fcf32210194", + "id": "fde4f550-d598-11ee-becf-dfae5e789f77", "type": "search", "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY3MiwxXQ==", + "updated_at": "2024-02-27T17:52:46.344Z", + "version": "Wzk2MywxXQ==", "attributes": { - "title": "PROFINET - Logs", + "title": "PROFINET I/O CM - Logs", "description": "", "hits": 0, "columns": [ @@ -409,27 +380,27 @@ "source.port", "destination.ip", "destination.port", - "zeek.profinet.block_version", - "zeek.profinet.operation_type", - "zeek.profinet.index", + "event.action", + "network.protocol_version", + "zeek.profinet_io_cm.sequence_num", "event.id" ], "sort": [ [ - "MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER", + "firstPacket", "desc" ] ], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"event.dataset:profinet\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"query\":{\"query\":\"event.dataset:profinet_io_cm\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"firstPacket\",\"fixed_interval\":\"30d\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ { "name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern", - "id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" + "id": "arkime_sessions3-*" } ], "migrationVersion": { @@ -437,15 +408,15 @@ } }, { - "id": "96d31d60-eba5-11e9-a384-0fcf32210194", + "id": "8524e670-eba5-11e9-a384-0fcf32210194", "type": "search", "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY3MywxXQ==", + "updated_at": "2024-02-27T17:29:03.431Z", + "version": "WzYxNCwxXQ==", "attributes": { - "title": "PROFINET DCE/RPC - Logs", + "title": "PROFINET - Logs", "description": "", "hits": 0, "columns": [ @@ -453,26 +424,27 @@ "source.port", "destination.ip", "destination.port", - "zeek.profinet_dce_rpc.version", - "zeek.profinet_dce_rpc.operation", + "zeek.profinet.block_version", + "zeek.profinet.operation_type", + "zeek.profinet.index", "event.id" ], "sort": [ [ - "MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER", + "firstPacket", "desc" ] ], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"event.dataset:profinet_dce_rpc\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"event.dataset:profinet\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ { "name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern", - "id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" + "id": "arkime_sessions3-*" } ], "migrationVersion": { From 09cf72d50b399dc5cc8348882ee75a60acc8db25 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 27 Feb 2024 12:00:48 -0700 Subject: [PATCH 43/79] work in progress incorporating icsnpp-profinet-io-cm, idaholab/Malcolm#429 --- logstash/pipelines/zeek/11_zeek_parse.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index 034c7e569..2193b6546 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -2709,6 +2709,13 @@ filter { split => { "[zeek_cols][sel_ack]" => "," } } } + if ([zeek_cols][packet_type]) and ((![zeek_cols][operation]) or ([zeek_cols][operation] == '(empty)') or + ([zeek_cols][operation] == 'unknown') or ([zeek_cols][operation] == '-') or + ([zeek_cols][operation] == '')) { + mutate { id => "mutate_replace_zeek_profinet_io_cm_operation" + replace => { "[zeek_cols][operation]" => "%{[zeek_cols][packet_type]}" } } + } + mutate { id => "mutate_add_fields_zeek_profinet_io_cm" add_field => { From 75a483ebeca62dd0a080413a3df430c0ba4d261d Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 27 Feb 2024 13:23:47 -0700 Subject: [PATCH 44/79] finished icsnpp-profinet-io-cm, idaholab/Malcolm#429 --- arkime/etc/config.ini | 5 ++-- arkime/wise/source.zeeklogs.js | 3 +-- .../composable/component/zeek_ot.json | 3 +-- logstash/pipelines/zeek/11_zeek_parse.conf | 7 ----- logstash/pipelines/zeek/12_zeek_mutate.conf | 26 +++++++++++++++++++ .../pipelines/zeek/13_zeek_normalize.conf | 5 +++- 6 files changed, 34 insertions(+), 15 deletions(-) diff --git a/arkime/etc/config.ini b/arkime/etc/config.ini index 7f3ff2290..0e9ef76d1 100644 --- a/arkime/etc/config.ini +++ b/arkime/etc/config.ini @@ -1506,8 +1506,7 @@ zeek.profinet_io_cm.object_uuid=db:zeek.profinet_io_cm.object_uuid;group:zeek_pr zeek.profinet_io_cm.interface_uuid=db:zeek.profinet_io_cm.interface_uuid;group:zeek_profinet_io_cm;kind:termfield;friendly:Interface UUID;help:Interface UUID zeek.profinet_io_cm.activity_uuid=db:zeek.profinet_io_cm.activity_uuid;group:zeek_profinet_io_cm;kind:termfield;friendly:Activity UUID;help:Activity UUID zeek.profinet_io_cm.server_boot_time=db:zeek.profinet_io_cm.server_boot_time;group:zeek_profinet_io_cm;kind:integer;friendly:Server boot time;help:Server boot time -zeek.profinet_io_cm.interface_vers_major=db:zeek.profinet_io_cm.interface_vers_major;group:zeek_profinet_io_cm;kind:integer;friendly:Interface version major;help:Interface version major -zeek.profinet_io_cm.interface_vers_minor=db:zeek.profinet_io_cm.interface_vers_minor;group:zeek_profinet_io_cm;kind:integer;friendly:Interface version minor;help:Interface version minor +zeek.profinet_io_cm.interface_version=db:zeek.profinet_io_cm.interface_version;group:zeek_profinet_io_cm;kind:termfield;friendly:Interface version;help:Interface version zeek.profinet_io_cm.sequence_num=db:zeek.profinet_io_cm.sequence_num;group:zeek_profinet_io_cm;kind:integer;friendly:Activity sequence number;help:Activity sequence number zeek.profinet_io_cm.operation=db:zeek.profinet_io_cm.operation;group:zeek_profinet_io_cm;kind:termfield;friendly:PNIO operation;help:PNIO operation zeek.profinet_io_cm.interface_hint=db:zeek.profinet_io_cm.interface_hint;group:zeek_profinet_io_cm;kind:integer;friendly:Interface hint;help:Interface hint @@ -2692,7 +2691,7 @@ o_zeek_ospf=require:zeek.ospf;title:Zeek ospf.log;fields:zeek.ospf.ospf_type,zee o_zeek_pe=require:zeek.pe;title:Zeek pe.log;fields:zeek.pe.machine,zeek.pe.compile_ts,zeek.pe.os,zeek.pe.subsystem,zeek.pe.is_exe,zeek.pe.is_64bit,zeek.pe.uses_aslr,zeek.pe.uses_dep,zeek.pe.uses_code_integrity,zeek.pe.uses_seh,zeek.pe.has_import_table,zeek.pe.has_export_table,zeek.pe.has_cert_table,zeek.pe.has_debug_data,zeek.pe.section_names o_zeek_profinet=require:zeek.profinet;title:Zeek profinet.log;fields:zeek.profinet.operation_type,zeek.profinet.block_version,zeek.profinet.slot_number,zeek.profinet.subslot_number,zeek.profinet.index o_zeek_profinet_dce_rpc=require:zeek.profinet_dce_rpc;title:Zeek profinet_dce_rpc.log;fields:zeek.profinet_dce_rpc.version,zeek.profinet_dce_rpc.packet_type,zeek.profinet_dce_rpc.object_uuid,zeek.profinet_dce_rpc.interface_uuid,zeek.profinet_dce_rpc.activity_uuid,zeek.profinet_dce_rpc.server_boot_time,zeek.profinet_dce_rpc.operation -o_zeek_profinet_io_cm=require:zeek.profinet_io_cm;title:Zeek profinet_io_cm.log;fields:zeek.profinet_io_cm.rpc_version,zeek.profinet_io_cm.packet_type,zeek.profinet_io_cm.reserved_for_impl_1,zeek.profinet_io_cm.last_fragment,zeek.profinet_io_cm.fragment,zeek.profinet_io_cm.no_fragment_requested,zeek.profinet_io_cm.maybe,zeek.profinet_io_cm.idempotent,zeek.profinet_io_cm.broadcast,zeek.profinet_io_cm.reserved_for_impl_2,zeek.profinet_io_cm.cancel_was_pending_at_call_end,zeek.profinet_io_cm.integer_encoding,zeek.profinet_io_cm.character_encoding,zeek.profinet_io_cm.floating_point_encoding,zeek.profinet_io_cm.serial_high,zeek.profinet_io_cm.object_uuid,zeek.profinet_io_cm.interface_uuid,zeek.profinet_io_cm.activity_uuid,zeek.profinet_io_cm.server_boot_time,zeek.profinet_io_cm.interface_vers_major,zeek.profinet_io_cm.interface_vers_minor,zeek.profinet_io_cm.sequence_num,zeek.profinet_io_cm.operation,zeek.profinet_io_cm.interface_hint,zeek.profinet_io_cm.activity_hint,zeek.profinet_io_cm.len_of_body,zeek.profinet_io_cm.fragment_num,zeek.profinet_io_cm.auth_protocol,zeek.profinet_io_cm.serial_low,zeek.profinet_io_cm.vers_fack,zeek.profinet_io_cm.window_size,zeek.profinet_io_cm.max_tsdu,zeek.profinet_io_cm.max_frag_size,zeek.profinet_io_cm.serial_number,zeek.profinet_io_cm.sel_ack_len,zeek.profinet_io_cm.sel_ack +o_zeek_profinet_io_cm=require:zeek.profinet_io_cm;title:Zeek profinet_io_cm.log;fields:zeek.profinet_io_cm.rpc_version,zeek.profinet_io_cm.packet_type,zeek.profinet_io_cm.reserved_for_impl_1,zeek.profinet_io_cm.last_fragment,zeek.profinet_io_cm.fragment,zeek.profinet_io_cm.no_fragment_requested,zeek.profinet_io_cm.maybe,zeek.profinet_io_cm.idempotent,zeek.profinet_io_cm.broadcast,zeek.profinet_io_cm.reserved_for_impl_2,zeek.profinet_io_cm.cancel_was_pending_at_call_end,zeek.profinet_io_cm.integer_encoding,zeek.profinet_io_cm.character_encoding,zeek.profinet_io_cm.floating_point_encoding,zeek.profinet_io_cm.serial_high,zeek.profinet_io_cm.object_uuid,zeek.profinet_io_cm.interface_uuid,zeek.profinet_io_cm.activity_uuid,zeek.profinet_io_cm.server_boot_time,zeek.profinet_io_cm.interface_version,zeek.profinet_io_cm.sequence_num,zeek.profinet_io_cm.operation,zeek.profinet_io_cm.interface_hint,zeek.profinet_io_cm.activity_hint,zeek.profinet_io_cm.len_of_body,zeek.profinet_io_cm.fragment_num,zeek.profinet_io_cm.auth_protocol,zeek.profinet_io_cm.serial_low,zeek.profinet_io_cm.vers_fack,zeek.profinet_io_cm.window_size,zeek.profinet_io_cm.max_tsdu,zeek.profinet_io_cm.max_frag_size,zeek.profinet_io_cm.serial_number,zeek.profinet_io_cm.sel_ack_len,zeek.profinet_io_cm.sel_ack o_zeek_radius=require:zeek.radius;title:Zeek radius.log;fields:zeek.radius.mac,zeek.radius.framed_addr,zeek.radius.tunnel_client,zeek.radius.connect_info,zeek.radius.reply_msg,zeek.radius.result,zeek.radius.ttl o_zeek_rdp=require:zeek.rdp;title:Zeek rdp.log;fields:zeek.rdp.cookie,zeek.rdp.result,zeek.rdp.security_protocol,zeek.rdp.client_channels,zeek.rdp.keyboard_layout,zeek.rdp.client_build,zeek.rdp.client_name,zeek.rdp.client_dig_product_id,zeek.rdp.desktop_width,zeek.rdp.desktop_height,zeek.rdp.requested_color_depth,zeek.rdp.cert_type,zeek.rdp.cert_count,zeek.rdp.cert_permanent,zeek.rdp.encryption_level,zeek.rdp.encryption_method o_zeek_rfb=require:zeek.rfb;title:Zeek rfb.log;fields:zeek.rfb.client_major_version,zeek.rfb.client_minor_version,zeek.rfb.server_major_version,zeek.rfb.server_minor_version,zeek.rfb.authentication_method,zeek.rfb.auth,zeek.rfb.share_flag,zeek.rfb.desktop_name,zeek.rfb.width,zeek.rfb.height diff --git a/arkime/wise/source.zeeklogs.js b/arkime/wise/source.zeeklogs.js index 1a47c019e..7015fa6c9 100644 --- a/arkime/wise/source.zeeklogs.js +++ b/arkime/wise/source.zeeklogs.js @@ -1738,8 +1738,7 @@ class MalcolmSource extends WISESource { "zeek.profinet_io_cm.interface_uuid", "zeek.profinet_io_cm.activity_uuid", "zeek.profinet_io_cm.server_boot_time", - "zeek.profinet_io_cm.interface_vers_major", - "zeek.profinet_io_cm.interface_vers_minor", + "zeek.profinet_io_cm.interface_version", "zeek.profinet_io_cm.sequence_num", "zeek.profinet_io_cm.operation_num", "zeek.profinet_io_cm.interface_hint", diff --git a/dashboards/templates/composable/component/zeek_ot.json b/dashboards/templates/composable/component/zeek_ot.json index 55917246f..58e4970a5 100644 --- a/dashboards/templates/composable/component/zeek_ot.json +++ b/dashboards/templates/composable/component/zeek_ot.json @@ -710,8 +710,7 @@ "zeek.profinet_io_cm.interface_uuid": { "type": "keyword" }, "zeek.profinet_io_cm.activity_uuid": { "type": "keyword" }, "zeek.profinet_io_cm.server_boot_time": { "type": "long" }, - "zeek.profinet_io_cm.interface_vers_major": { "type": "long" }, - "zeek.profinet_io_cm.interface_vers_minor": { "type": "long" }, + "zeek.profinet_io_cm.interface_version": { "type": "keyword" }, "zeek.profinet_io_cm.sequence_num": { "type": "long" }, "zeek.profinet_io_cm.operation": { "type": "keyword" }, "zeek.profinet_io_cm.interface_hint": { "type": "long" }, diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index 2193b6546..034c7e569 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -2709,13 +2709,6 @@ filter { split => { "[zeek_cols][sel_ack]" => "," } } } - if ([zeek_cols][packet_type]) and ((![zeek_cols][operation]) or ([zeek_cols][operation] == '(empty)') or - ([zeek_cols][operation] == 'unknown') or ([zeek_cols][operation] == '-') or - ([zeek_cols][operation] == '')) { - mutate { id => "mutate_replace_zeek_profinet_io_cm_operation" - replace => { "[zeek_cols][operation]" => "%{[zeek_cols][packet_type]}" } } - } - mutate { id => "mutate_add_fields_zeek_profinet_io_cm" add_field => { diff --git a/logstash/pipelines/zeek/12_zeek_mutate.conf b/logstash/pipelines/zeek/12_zeek_mutate.conf index 4ea9f12b0..9c15300ca 100644 --- a/logstash/pipelines/zeek/12_zeek_mutate.conf +++ b/logstash/pipelines/zeek/12_zeek_mutate.conf @@ -1380,6 +1380,32 @@ filter { add_field => { "[zeek][software][software_type]" => "OPCUA" } } } + } else if ([log_source] == "profinet_io_cm") { + ############################################################################################################################# + # profinet_io_cm.log specific logic + + if ([zeek][profinet_io_cm][packet_type]) and + ((![zeek][profinet_io_cm][operation]) or ([zeek][profinet_io_cm][operation] == '(empty)') or + ([zeek][profinet_io_cm][operation] == 'unknown') or ([zeek][profinet_io_cm][operation] == '-') or + ([zeek][profinet_io_cm][operation] == '')) { + mutate { id => "mutate_replace_zeek_profinet_io_cm_operation" + replace => { "[zeek][profinet_io_cm][operation]" => "%{[zeek][profinet_io_cm][packet_type]}" } } + } + + if ([zeek][profinet_io_cm][interface_vers_major]) or ([zeek][profinet_io_cm][interface_vers_minor]) { + ruby { + id => "ruby_zeek_profinet_io_cm_interface_vers" + code => ' + event.set("[zeek][profinet_io_cm][interface_version]", + [event.get("[zeek][profinet_io_cm][interface_vers_major]").to_s, + event.get("[zeek][profinet_io_cm][interface_vers_minor]").to_s].join(".")) + ' + } + mutate { id => "mutate_remove_fields_zeek_profinet_io_cm_interface_vers" + remove_field => [ "[zeek][profinet_io_cm][interface_vers_major]", + "[zeek][profinet_io_cm][interface_vers_minor]" ] } + } + } else if ([log_source] == "radius") { ############################################################################################################################# # radius.log specific logic diff --git a/logstash/pipelines/zeek/13_zeek_normalize.conf b/logstash/pipelines/zeek/13_zeek_normalize.conf index 8c0d62063..17cc0076b 100644 --- a/logstash/pipelines/zeek/13_zeek_normalize.conf +++ b/logstash/pipelines/zeek/13_zeek_normalize.conf @@ -115,9 +115,12 @@ filter { if ([zeek][profinet_dce_rpc][version]) { mutate { id => "mutate_merge_normalize_zeek_profinet_dce_rpc_version" merge => { "[network][protocol_version]" => "[zeek][profinet_dce_rpc][version]" } } } - if ([zeek][profinet_io_cm][rpc_version]) { mutate { id => "mutate_merge_normalize_zeek_profinet_io_cm_version" + if ([zeek][profinet_io_cm][rpc_version]) { mutate { id => "mutate_merge_normalize_zeek_profinet_io_cm_rpc_version" merge => { "[network][protocol_version]" => "[zeek][profinet_io_cm][rpc_version]" } } } + if ([zeek][profinet_io_cm][interface_version]) { mutate { id => "mutate_merge_normalize_zeek_profinet_io_interface_version" + merge => { "[network][protocol_version]" => "[zeek][profinet_io_cm][interface_version]" } } } + if ([zeek][rfb]) { ruby { id => "ruby_zeek_field_network_protocol_version_rfb" From 3a365031acb6f708d6fe4506af62a8e9216688b2 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 27 Feb 2024 15:11:40 -0700 Subject: [PATCH 45/79] interface_version -> uuid_version for profinet I/O CM --- arkime/etc/config.ini | 4 ++-- arkime/wise/source.zeeklogs.js | 2 +- dashboards/templates/composable/component/zeek_ot.json | 2 +- logstash/pipelines/zeek/12_zeek_mutate.conf | 2 +- logstash/pipelines/zeek/13_zeek_normalize.conf | 3 --- 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/arkime/etc/config.ini b/arkime/etc/config.ini index 0e9ef76d1..4cb262d09 100644 --- a/arkime/etc/config.ini +++ b/arkime/etc/config.ini @@ -1506,7 +1506,7 @@ zeek.profinet_io_cm.object_uuid=db:zeek.profinet_io_cm.object_uuid;group:zeek_pr zeek.profinet_io_cm.interface_uuid=db:zeek.profinet_io_cm.interface_uuid;group:zeek_profinet_io_cm;kind:termfield;friendly:Interface UUID;help:Interface UUID zeek.profinet_io_cm.activity_uuid=db:zeek.profinet_io_cm.activity_uuid;group:zeek_profinet_io_cm;kind:termfield;friendly:Activity UUID;help:Activity UUID zeek.profinet_io_cm.server_boot_time=db:zeek.profinet_io_cm.server_boot_time;group:zeek_profinet_io_cm;kind:integer;friendly:Server boot time;help:Server boot time -zeek.profinet_io_cm.interface_version=db:zeek.profinet_io_cm.interface_version;group:zeek_profinet_io_cm;kind:termfield;friendly:Interface version;help:Interface version +zeek.profinet_io_cm.uuid_version=db:zeek.profinet_io_cm.uuid_version;group:zeek_profinet_io_cm;kind:termfield;friendly:Interface version;help:UUID version zeek.profinet_io_cm.sequence_num=db:zeek.profinet_io_cm.sequence_num;group:zeek_profinet_io_cm;kind:integer;friendly:Activity sequence number;help:Activity sequence number zeek.profinet_io_cm.operation=db:zeek.profinet_io_cm.operation;group:zeek_profinet_io_cm;kind:termfield;friendly:PNIO operation;help:PNIO operation zeek.profinet_io_cm.interface_hint=db:zeek.profinet_io_cm.interface_hint;group:zeek_profinet_io_cm;kind:integer;friendly:Interface hint;help:Interface hint @@ -2691,7 +2691,7 @@ o_zeek_ospf=require:zeek.ospf;title:Zeek ospf.log;fields:zeek.ospf.ospf_type,zee o_zeek_pe=require:zeek.pe;title:Zeek pe.log;fields:zeek.pe.machine,zeek.pe.compile_ts,zeek.pe.os,zeek.pe.subsystem,zeek.pe.is_exe,zeek.pe.is_64bit,zeek.pe.uses_aslr,zeek.pe.uses_dep,zeek.pe.uses_code_integrity,zeek.pe.uses_seh,zeek.pe.has_import_table,zeek.pe.has_export_table,zeek.pe.has_cert_table,zeek.pe.has_debug_data,zeek.pe.section_names o_zeek_profinet=require:zeek.profinet;title:Zeek profinet.log;fields:zeek.profinet.operation_type,zeek.profinet.block_version,zeek.profinet.slot_number,zeek.profinet.subslot_number,zeek.profinet.index o_zeek_profinet_dce_rpc=require:zeek.profinet_dce_rpc;title:Zeek profinet_dce_rpc.log;fields:zeek.profinet_dce_rpc.version,zeek.profinet_dce_rpc.packet_type,zeek.profinet_dce_rpc.object_uuid,zeek.profinet_dce_rpc.interface_uuid,zeek.profinet_dce_rpc.activity_uuid,zeek.profinet_dce_rpc.server_boot_time,zeek.profinet_dce_rpc.operation -o_zeek_profinet_io_cm=require:zeek.profinet_io_cm;title:Zeek profinet_io_cm.log;fields:zeek.profinet_io_cm.rpc_version,zeek.profinet_io_cm.packet_type,zeek.profinet_io_cm.reserved_for_impl_1,zeek.profinet_io_cm.last_fragment,zeek.profinet_io_cm.fragment,zeek.profinet_io_cm.no_fragment_requested,zeek.profinet_io_cm.maybe,zeek.profinet_io_cm.idempotent,zeek.profinet_io_cm.broadcast,zeek.profinet_io_cm.reserved_for_impl_2,zeek.profinet_io_cm.cancel_was_pending_at_call_end,zeek.profinet_io_cm.integer_encoding,zeek.profinet_io_cm.character_encoding,zeek.profinet_io_cm.floating_point_encoding,zeek.profinet_io_cm.serial_high,zeek.profinet_io_cm.object_uuid,zeek.profinet_io_cm.interface_uuid,zeek.profinet_io_cm.activity_uuid,zeek.profinet_io_cm.server_boot_time,zeek.profinet_io_cm.interface_version,zeek.profinet_io_cm.sequence_num,zeek.profinet_io_cm.operation,zeek.profinet_io_cm.interface_hint,zeek.profinet_io_cm.activity_hint,zeek.profinet_io_cm.len_of_body,zeek.profinet_io_cm.fragment_num,zeek.profinet_io_cm.auth_protocol,zeek.profinet_io_cm.serial_low,zeek.profinet_io_cm.vers_fack,zeek.profinet_io_cm.window_size,zeek.profinet_io_cm.max_tsdu,zeek.profinet_io_cm.max_frag_size,zeek.profinet_io_cm.serial_number,zeek.profinet_io_cm.sel_ack_len,zeek.profinet_io_cm.sel_ack +o_zeek_profinet_io_cm=require:zeek.profinet_io_cm;title:Zeek profinet_io_cm.log;fields:zeek.profinet_io_cm.rpc_version,zeek.profinet_io_cm.packet_type,zeek.profinet_io_cm.reserved_for_impl_1,zeek.profinet_io_cm.last_fragment,zeek.profinet_io_cm.fragment,zeek.profinet_io_cm.no_fragment_requested,zeek.profinet_io_cm.maybe,zeek.profinet_io_cm.idempotent,zeek.profinet_io_cm.broadcast,zeek.profinet_io_cm.reserved_for_impl_2,zeek.profinet_io_cm.cancel_was_pending_at_call_end,zeek.profinet_io_cm.integer_encoding,zeek.profinet_io_cm.character_encoding,zeek.profinet_io_cm.floating_point_encoding,zeek.profinet_io_cm.serial_high,zeek.profinet_io_cm.object_uuid,zeek.profinet_io_cm.interface_uuid,zeek.profinet_io_cm.activity_uuid,zeek.profinet_io_cm.server_boot_time,zeek.profinet_io_cm.uuid_version,zeek.profinet_io_cm.sequence_num,zeek.profinet_io_cm.operation,zeek.profinet_io_cm.interface_hint,zeek.profinet_io_cm.activity_hint,zeek.profinet_io_cm.len_of_body,zeek.profinet_io_cm.fragment_num,zeek.profinet_io_cm.auth_protocol,zeek.profinet_io_cm.serial_low,zeek.profinet_io_cm.vers_fack,zeek.profinet_io_cm.window_size,zeek.profinet_io_cm.max_tsdu,zeek.profinet_io_cm.max_frag_size,zeek.profinet_io_cm.serial_number,zeek.profinet_io_cm.sel_ack_len,zeek.profinet_io_cm.sel_ack o_zeek_radius=require:zeek.radius;title:Zeek radius.log;fields:zeek.radius.mac,zeek.radius.framed_addr,zeek.radius.tunnel_client,zeek.radius.connect_info,zeek.radius.reply_msg,zeek.radius.result,zeek.radius.ttl o_zeek_rdp=require:zeek.rdp;title:Zeek rdp.log;fields:zeek.rdp.cookie,zeek.rdp.result,zeek.rdp.security_protocol,zeek.rdp.client_channels,zeek.rdp.keyboard_layout,zeek.rdp.client_build,zeek.rdp.client_name,zeek.rdp.client_dig_product_id,zeek.rdp.desktop_width,zeek.rdp.desktop_height,zeek.rdp.requested_color_depth,zeek.rdp.cert_type,zeek.rdp.cert_count,zeek.rdp.cert_permanent,zeek.rdp.encryption_level,zeek.rdp.encryption_method o_zeek_rfb=require:zeek.rfb;title:Zeek rfb.log;fields:zeek.rfb.client_major_version,zeek.rfb.client_minor_version,zeek.rfb.server_major_version,zeek.rfb.server_minor_version,zeek.rfb.authentication_method,zeek.rfb.auth,zeek.rfb.share_flag,zeek.rfb.desktop_name,zeek.rfb.width,zeek.rfb.height diff --git a/arkime/wise/source.zeeklogs.js b/arkime/wise/source.zeeklogs.js index 7015fa6c9..140ec922f 100644 --- a/arkime/wise/source.zeeklogs.js +++ b/arkime/wise/source.zeeklogs.js @@ -1738,7 +1738,7 @@ class MalcolmSource extends WISESource { "zeek.profinet_io_cm.interface_uuid", "zeek.profinet_io_cm.activity_uuid", "zeek.profinet_io_cm.server_boot_time", - "zeek.profinet_io_cm.interface_version", + "zeek.profinet_io_cm.uuid_version", "zeek.profinet_io_cm.sequence_num", "zeek.profinet_io_cm.operation_num", "zeek.profinet_io_cm.interface_hint", diff --git a/dashboards/templates/composable/component/zeek_ot.json b/dashboards/templates/composable/component/zeek_ot.json index 58e4970a5..fb09b02e0 100644 --- a/dashboards/templates/composable/component/zeek_ot.json +++ b/dashboards/templates/composable/component/zeek_ot.json @@ -710,7 +710,7 @@ "zeek.profinet_io_cm.interface_uuid": { "type": "keyword" }, "zeek.profinet_io_cm.activity_uuid": { "type": "keyword" }, "zeek.profinet_io_cm.server_boot_time": { "type": "long" }, - "zeek.profinet_io_cm.interface_version": { "type": "keyword" }, + "zeek.profinet_io_cm.uuid_version": { "type": "keyword" }, "zeek.profinet_io_cm.sequence_num": { "type": "long" }, "zeek.profinet_io_cm.operation": { "type": "keyword" }, "zeek.profinet_io_cm.interface_hint": { "type": "long" }, diff --git a/logstash/pipelines/zeek/12_zeek_mutate.conf b/logstash/pipelines/zeek/12_zeek_mutate.conf index 9c15300ca..06b74ee15 100644 --- a/logstash/pipelines/zeek/12_zeek_mutate.conf +++ b/logstash/pipelines/zeek/12_zeek_mutate.conf @@ -1396,7 +1396,7 @@ filter { ruby { id => "ruby_zeek_profinet_io_cm_interface_vers" code => ' - event.set("[zeek][profinet_io_cm][interface_version]", + event.set("[zeek][profinet_io_cm][uuid_version]", [event.get("[zeek][profinet_io_cm][interface_vers_major]").to_s, event.get("[zeek][profinet_io_cm][interface_vers_minor]").to_s].join(".")) ' diff --git a/logstash/pipelines/zeek/13_zeek_normalize.conf b/logstash/pipelines/zeek/13_zeek_normalize.conf index 17cc0076b..917c9198b 100644 --- a/logstash/pipelines/zeek/13_zeek_normalize.conf +++ b/logstash/pipelines/zeek/13_zeek_normalize.conf @@ -118,9 +118,6 @@ filter { if ([zeek][profinet_io_cm][rpc_version]) { mutate { id => "mutate_merge_normalize_zeek_profinet_io_cm_rpc_version" merge => { "[network][protocol_version]" => "[zeek][profinet_io_cm][rpc_version]" } } } - if ([zeek][profinet_io_cm][interface_version]) { mutate { id => "mutate_merge_normalize_zeek_profinet_io_interface_version" - merge => { "[network][protocol_version]" => "[zeek][profinet_io_cm][interface_version]" } } } - if ([zeek][rfb]) { ruby { id => "ruby_zeek_field_network_protocol_version_rfb" From 96f1cc2ded30d0981059bbd00f0550ca24faa324 Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 28 Feb 2024 08:30:23 -0700 Subject: [PATCH 46/79] documentation udpates for idaholab/Malcolm#336, expand MISP usage --- docs/zeek-intel.md | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/zeek-intel.md b/docs/zeek-intel.md index 20cd52427..6b0704ed6 100644 --- a/docs/zeek-intel.md +++ b/docs/zeek-intel.md @@ -8,9 +8,9 @@ To quote Zeek's [Intelligence Framework](https://docs.zeek.org/en/master/framewo Malcolm doesn't come bundled with intelligence files from any particular feed, but they can be easily included into a local instance. On [startup]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/zeek_intel_setup.sh), Malcolm's `ghcr.io/idaholab/malcolm/zeek` docker container enumerates the subdirectories under `./zeek/intel` (which is [bind mounted](https://docs.docker.com/storage/bind-mounts/) into the container's runtime) and configures Zeek so those intelligence files will be automatically included in its local policy. Subdirectories under `./zeek/intel` that contain their own `__load__.zeek` file will be `@load`-ed as-is, while subdirectories containing "loose" intelligence files will be [loaded](https://docs.zeek.org/en/master/frameworks/intel.html#loading-intelligence) automatically with a `redef Intel::read_files` directive. -Note that Malcolm does not manage updates for these intelligence files. You should use the update mechanism suggested by the feeds' maintainers to keep intelligence files up to date, or use a [TAXII](#ZeekIntelSTIX) or [MISP](#ZeekIntelMISP) feed as described below. +Note that Malcolm does not manage updates for these intelligence files. Users use the update mechanism suggested by the feeds' maintainers to keep intelligence files up to date, or use a [TAXII](#ZeekIntelSTIX) or [MISP](#ZeekIntelMISP) feed as described below. -Adding and deleting intelligence files under this directory will take effect upon [restarting Malcolm](running.md#StopAndRestart). Alternately, you can use the `ZEEK_INTEL_REFRESH_CRON_EXPRESSION` environment variable containing a [cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression) to specify the interval at which the intel files should be refreshed. This can also be done manually without restarting Malcolm by running the following command from the Malcolm installation directory: +Adding and deleting intelligence files under this directory will take effect upon [restarting Malcolm](running.md#StopAndRestart). Alternately, users can use the `ZEEK_INTEL_REFRESH_CRON_EXPRESSION` environment variable containing a [cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression) to specify the interval at which the intel files should be refreshed. This can also be done manually without restarting Malcolm by running the following command from the Malcolm installation directory: ``` docker compose exec --user $(id -u) zeek /usr/local/bin/entrypoint.sh true @@ -36,7 +36,7 @@ taxii|2.1|https://example.com/taxii/api2/|URL Blocklist … ``` -Malcolm will attempt to query the TAXII feed(s) for `indicator` STIX objects and convert them to the Zeek intelligence format as described above. There are publicly available TAXII 2.x-compatible services provided by a number of organizations including [Anomali Labs](https://www.anomali.com/resources/limo) and [MITRE](https://www.mitre.org/capabilities/cybersecurity/overview/cybersecurity-blog/attck%E2%84%A2-content-available-in-stix%E2%84%A2-20-via); or you may choose from several open-source offerings to roll your own TAXII 2 server (e.g., [oasis-open/cti-taxii-server](https://github.com/oasis-open/cti-taxii-server), [freetaxii/server](https://github.com/freetaxii/server), [StephenOTT/TAXII-Server](https://github.com/StephenOTT/TAXII-Server), etc.). +Malcolm will attempt to query the TAXII feed(s) for `indicator` STIX objects and convert them to the Zeek intelligence format as described above. There are publicly available TAXII 2.x-compatible services provided by a number of organizations including [Anomali Labs](https://www.anomali.com/resources/limo) and [MITRE](https://www.mitre.org/capabilities/cybersecurity/overview/cybersecurity-blog/attck%E2%84%A2-content-available-in-stix%E2%84%A2-20-via); or users may choose from several open-source offerings to roll their own TAXII 2 server (e.g., [oasis-open/cti-taxii-server](https://github.com/oasis-open/cti-taxii-server), [freetaxii/server](https://github.com/freetaxii/server), [StephenOTT/TAXII-Server](https://github.com/StephenOTT/TAXII-Server), etc.). Note that only **indicators** of [**cyber-observable objects**](https://docs.oasis-open.org/cti/stix/v2.1/cs01/stix-v2.1-cs01.html#_mlbmudhl16lr) matched with the **equals (`=`)** [comparison operator](https://docs.oasis-open.org/cti/stix/v2.1/cs01/stix-v2.1-cs01.html#_t11hn314cr7w) against a **single value** can be expressed as Zeek intelligence items. More complex STIX indicators will be silently ignored. @@ -44,19 +44,28 @@ Note that only **indicators** of [**cyber-observable objects**](https://docs.oas In addition to loading Zeek intelligence files on startup, Malcolm will [automatically generate]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/zeek_intel_from_threat_feed.py) a Zeek intelligence file for all [Malware Information Sharing Platform (MISP)](https://www.misp-project.org/datamodels/) JSON files found under `./zeek/intel/MISP`. -Additionally, if a special text file named `.misp_input.txt` is found in `./zeek/intel/MISP`, that file will be read and processed as a list of [MISP feed](https://misp.gitbooks.io/misp-book/content/managing-feeds/#feeds) URLs, one per line, according to the following format (the authentication key is optional): +Additionally, if a special text file named `.misp_input.txt` is found in `./zeek/intel/MISP`, that file will be read and processed as a list of [MISP feed](https://misp.gitbooks.io/misp-book/content/managing-feeds/#feeds) URLs, one per line, according to the following format: ``` -misp|manifest_url|auth_key +misp|misp_url|auth_key (optional) ``` For example: ``` misp|https://example.com/data/feed-osint/manifest.json|df97338db644c64fbfd90f3e03ba8870 +misp|https://example.com/doc/misp/| +misp|https://example.com/attributes|a943f5ff506ee6198e996333e0b672b1 … ``` -Malcolm will attempt to connect to the MISP feed(s) and retrieve [`Attribute`](https://www.misp-standard.org/rfc/misp-standard-core.html#name-attribute) objects of MISP events and convert them to the Zeek intelligence format as described above. There are publicly available [MISP feeds](https://www.misp-project.org/feeds/) and [communities](https://www.misp-project.org/communities/), or you may [run your own MISP instance](https://www.misp-project.org/2019/09/25/hostev-vs-own-misp.html/). +Malcolm will attempt to connect to the MISP feed(s) and retrieve [`Attribute`](https://www.misp-standard.org/rfc/misp-standard-core.html#name-attribute) objects of MISP events and convert them to the Zeek intelligence format as described above. There are publicly available [MISP feeds](https://www.misp-project.org/feeds/) and [communities](https://www.misp-project.org/communities/), or users [may run](https://github.com/MISP/misp-docker) their [own MISP instance](https://www.misp-project.org/2019/09/25/hostev-vs-own-misp.html/). + +Upon Malcolm connects to the URLs for the MISP feeds in `.misp_input.txt`, it will attempt to determine the format of the data served and process it accordingly. This could be presented as: + +* a manifest JSON file +* a directory listing containing a file named `manifest.json` +* a directory listing of JSON files without a `manifest.json` file +* an array of [attributes](https://www.misp-project.org/openapi/#tag/Attributes) returned for a request via the [MISP Automation API](https://www.misp-project.org/openapi/) made to a MISP platform's `/attributes` endpoint Note that only a subset of MISP [attribute types](https://www.misp-project.org/datamodels/#attribute-categories-vs-types) can be expressed with the Zeek intelligence [indicator types](https://docs.zeek.org/en/master/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Type). MISP attributes with other types will be silently ignored. \ No newline at end of file From 95e8fc81d3ecb6f0d4f0007e4a2c2db3d4dee27c Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 28 Feb 2024 09:02:04 -0700 Subject: [PATCH 47/79] work in progress for idaholab/Malcolm#336, expand MISP usage --- Dockerfiles/zeek.Dockerfile | 2 ++ config/zeek.env.example | 2 ++ shared/bin/zeek_intel_from_threat_feed.py | 38 +++++++++++++++-------- shared/bin/zeek_intel_setup.sh | 2 ++ shared/bin/zeek_threat_feed_utils.py | 35 +++++++++++++++------ 5 files changed, 56 insertions(+), 23 deletions(-) diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index 63f18408c..986454342 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -199,6 +199,7 @@ ARG ZEEK_INTEL_ITEM_EXPIRATION=-1min ARG ZEEK_INTEL_REFRESH_CRON_EXPRESSION= ARG ZEEK_INTEL_REFRESH_THREADS=2 ARG ZEEK_INTEL_FEED_SINCE= +ARG ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION=false ARG ZEEK_EXTRACTOR_MODE=none ARG ZEEK_EXTRACTOR_PATH=/zeek/extract_files ARG ZEEK_INTEL_PATH=/opt/zeek/share/zeek/site/intel @@ -222,6 +223,7 @@ ENV ZEEK_INTEL_ITEM_EXPIRATION $ZEEK_INTEL_ITEM_EXPIRATION ENV ZEEK_INTEL_REFRESH_CRON_EXPRESSION $ZEEK_INTEL_REFRESH_CRON_EXPRESSION ENV ZEEK_INTEL_REFRESH_THREADS $ZEEK_INTEL_REFRESH_THREADS ENV ZEEK_INTEL_FEED_SINCE $ZEEK_INTEL_FEED_SINCE +eNV ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION $ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION ENV ZEEK_EXTRACTOR_MODE $ZEEK_EXTRACTOR_MODE ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH ENV ZEEK_INTEL_PATH $ZEEK_INTEL_PATH diff --git a/config/zeek.env.example b/config/zeek.env.example index a0bd28d98..400362120 100644 --- a/config/zeek.env.example +++ b/config/zeek.env.example @@ -11,6 +11,8 @@ ZEEK_INTEL_ITEM_EXPIRATION=-1min # been created or modified since the time represented by this value; # it may be either a fixed date/time (01/01/2021) or relative interval (30 days ago) ZEEK_INTEL_FEED_SINCE= +# Whether or not to require SSL certificate verification when querying a TAXII or MISP feed +ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION=false # Specifies a cron expression indicating the refresh interval for generating the # Zeek Intelligence Framework files ('' disables automatic refresh) ZEEK_INTEL_REFRESH_CRON_EXPRESSION= diff --git a/shared/bin/zeek_intel_from_threat_feed.py b/shared/bin/zeek_intel_from_threat_feed.py index 456387238..b2c4e6c72 100755 --- a/shared/bin/zeek_intel_from_threat_feed.py +++ b/shared/bin/zeek_intel_from_threat_feed.py @@ -31,7 +31,7 @@ def main(): [ 'Outputs a Zeek intelligence framework file from various formats used to represent threat information:', ' - "Indicator" objects in STIX™ v2.0/v2.1 JSON files', - ' - MISP core format JSON files', + ' - MISP attributes or core format JSON files', '', 'See:', ' - Malcolm documentation: https://idaholab.github.io/Malcolm/docs/zeek-intel.html#ZeekIntel', @@ -44,6 +44,7 @@ def main(): ' - MISP communities: https://www.misp-project.org/communities/', ' - MISP default feeds: https://www.misp-project.org/feeds/', ' - Managing MISP feeds: https://misp.gitbooks.io/misp-book/content/managing-feeds/', + ' - Expand MISP usage: https://github.com/idaholab/Malcolm/issues/336', '', 'Note: The Zeek intelligence framework only supports simple indicators matched against a single value.', 'The STIX™ standard can express more complex indicators that cannot be expressed with Zeek intelligence items.', @@ -53,30 +54,40 @@ def main(): add_help=False, usage='{} '.format(script_name), ) - parser.add_argument('--verbose', '-v', action='count', default=1, help='Increase verbosity (e.g., -v, -vv, etc.)') parser.add_argument( - '--notice', dest='notice', action='store_true', help='Add fields for policy/frameworks/intel/do_notice.zeek' + '--verbose', + '-v', + action='count', + default=1, + help='Increase verbosity (e.g., -v, -vv, etc.)', ) parser.add_argument( - '--no-notice', + '--notice', dest='notice', - action='store_false', - help='Do not add fields for policy/frameworks/intel/do_notice.zeek', + type=malcolm_utils.str2bool, + nargs='?', + const=True, + default=True, + help='Add fields for policy/frameworks/intel/do_notice.zeek', ) - parser.set_defaults(notice=True) parser.add_argument( '--cif', dest='cif', - action='store_true', + type=malcolm_utils.str2bool, + nargs='?', + const=True, + default=True, help='Add fields for policy/integration/collective-intel/main.zeek', ) parser.add_argument( - '--no-cif', - dest='cif', - action='store_false', - help='Do not add fields for policy/integration/collective-intel/main.zeek', + '--ssl-verify', + dest='sslVerify', + type=malcolm_utils.str2bool, + nargs='?', + const=True, + default=False, + help='Require TLS connections to verify certificates', ) - parser.set_defaults(cif=True) parser.add_argument( '-i', '--input', @@ -190,6 +201,7 @@ def main(): inputQueue, zeekPrinter, since, + args.sslVerify, defaultNow, workerThreadCount, logging, diff --git a/shared/bin/zeek_intel_setup.sh b/shared/bin/zeek_intel_setup.sh index d4a8ad80b..f2309e352 100755 --- a/shared/bin/zeek_intel_setup.sh +++ b/shared/bin/zeek_intel_setup.sh @@ -15,6 +15,7 @@ SCRIPT_FILESPEC="$(realpath -e "${BASH_SOURCE[0]}")" ZEEK_DIR=${ZEEK_DIR:-"/opt/zeek"} ZEEK_INTEL_ITEM_EXPIRATION=${ZEEK_INTEL_ITEM_EXPIRATION:-"-1min"} ZEEK_INTEL_FEED_SINCE=${ZEEK_INTEL_FEED_SINCE:-""} +ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION=${ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION:-false} ZEEK_INTEL_REFRESH_THREADS=${ZEEK_INTEL_REFRESH_THREADS:-"2"} INTEL_DIR=${INTEL_DIR:-"${ZEEK_DIR}/share/zeek/site/intel"} INTEL_PRESEED_DIR=${INTEL_PRESEED_DIR:-"${ZEEK_DIR}/share/zeek/site/intel-preseed"} @@ -92,6 +93,7 @@ EOF # process STIX and MISP inputs by converting them to Zeek intel format if ( (( ${#THREAT_JSON_FILES[@]} )) || [[ -r ./STIX/.stix_input.txt ]] || [[ -r ./MISP/.misp_input.txt ]] ) && [[ -x "${THREAT_FEED_TO_ZEEK_SCRIPT}" ]]; then "${THREAT_FEED_TO_ZEEK_SCRIPT}" \ + --ssl-verify ${ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION} \ --since "${ZEEK_INTEL_FEED_SINCE}" \ --threads ${ZEEK_INTEL_REFRESH_THREADS} \ --output ./.threat_autogen.zeek.new \ diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py index a7b0c21a7..813e9a8bb 100644 --- a/shared/bin/zeek_threat_feed_utils.py +++ b/shared/bin/zeek_threat_feed_utils.py @@ -34,6 +34,7 @@ import os import re import requests +import urllib3 from malcolm_utils import base64_decode_if_prefixed, LoadStrIfJson, LoadFileIfJson @@ -111,6 +112,8 @@ "x509-fingerprint-sha1": "CERT_HASH", } +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + # get URL directory listing def get_url_paths_from_response(response_text, parent_url='', ext=''): @@ -555,13 +558,14 @@ def ProcessMISP( def ProcessThreatInputWorker(threatInputWorkerArgs): - inputQueue, zeekPrinter, since, defaultNow, workerThreadCount, logger = ( + inputQueue, zeekPrinter, since, sslVerify, defaultNow, workerThreadCount, logger = ( threatInputWorkerArgs[0], threatInputWorkerArgs[1], threatInputWorkerArgs[2], threatInputWorkerArgs[3], threatInputWorkerArgs[4], threatInputWorkerArgs[5], + threatInputWorkerArgs[6], ) with workerThreadCount as workerId: @@ -626,7 +630,8 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): # - a manifest JSON (https://www.circl.lu/doc/misp/feed-osint/manifest.json) # - a directory listing *containing* a manifest.json (https://www.circl.lu/doc/misp/feed-osint/) # - a directory listing of misc. JSON files without a manifest.json - mispResponse = mispSession.get(mispUrl) + # - an array of attributes returned for a request via the MISP Automation API to an /attributes endpoint + mispResponse = mispSession.get(mispUrl, verify=sslVerify) mispResponse.raise_for_status() if mispJson := LoadStrIfJson(mispResponse.content): # the contents are JSON. determine if this is a manifest or a single event @@ -652,7 +657,7 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): else defaultNow ) if (since is None) or (eventTime >= since): - mispObjectReponse = mispSession.get(newUrl) + mispObjectReponse = mispSession.get(newUrl, verify=sslVerify) mispObjectReponse.raise_for_status() zeekPrinter.ProcessMISP( mispObjectReponse.json(), @@ -679,7 +684,7 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): # retrieve it, then loop over it and retrieve and process the MISP events it references for url in manifestPaths: try: - mispManifestResponse = mispSession.get(url) + mispManifestResponse = mispSession.get(url, verify=sslVerify) mispManifestResponse.raise_for_status() mispManifest = mispManifestResponse.json() for uri in mispManifest: @@ -693,7 +698,9 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): ) if (since is None) or (eventTime >= since): newUrl = f'{mispUrl.strip("/")}/{uri}.json' - mispObjectReponse = mispSession.get(newUrl) + mispObjectReponse = mispSession.get( + newUrl, verify=sslVerify + ) mispObjectReponse.raise_for_status() zeekPrinter.ProcessMISP( mispObjectReponse.json(), @@ -713,7 +720,7 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): # just loop over, retrieve and process the .json files in this directory for url in paths: try: - mispObjectReponse = mispSession.get(url) + mispObjectReponse = mispSession.get(url, verify=sslVerify) mispObjectReponse.raise_for_status() zeekPrinter.ProcessMISP( mispObjectReponse.json(), @@ -755,9 +762,13 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): # connect to the server with the appropriate API for the TAXII version if taxiiVersion == '2.0': - server = TaxiiServer_v20(taxiiDisoveryURL, user=taxiiUsername, password=taxiiPassword) + server = TaxiiServer_v20( + taxiiDisoveryURL, user=taxiiUsername, password=taxiiPassword, verify=sslVerify + ) elif taxiiVersion == '2.1': - server = TaxiiServer_v21(taxiiDisoveryURL, user=taxiiUsername, password=taxiiPassword) + server = TaxiiServer_v21( + taxiiDisoveryURL, user=taxiiUsername, password=taxiiPassword, verify=sslVerify + ) else: raise Exception(f"Unsupported TAXII version '{taxiiVersion}'") @@ -776,9 +787,13 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): # connect to and retrieve indicator STIX objects from the collection URL(s) for title, info in collectionUrls.items(): collection = ( - TaxiiCollection_v21(info['url'], user=taxiiUsername, password=taxiiPassword) + TaxiiCollection_v21( + info['url'], user=taxiiUsername, password=taxiiPassword, verify=sslVerify + ) if taxiiVersion == '2.1' - else TaxiiCollection_v20(info['url'], user=taxiiUsername, password=taxiiPassword) + else TaxiiCollection_v20( + info['url'], user=taxiiUsername, password=taxiiPassword, verify=sslVerify + ) ) try: # loop over paginated results From e9277b7f19d53275e1efecf58a85e681b20a3210 Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 28 Feb 2024 10:01:58 -0700 Subject: [PATCH 48/79] work in progress for idaholab/Malcolm#336, expand MISP usage --- shared/bin/zeek_intel_from_threat_feed.py | 1 + shared/bin/zeek_threat_feed_utils.py | 218 +++++++++++++++------- 2 files changed, 148 insertions(+), 71 deletions(-) diff --git a/shared/bin/zeek_intel_from_threat_feed.py b/shared/bin/zeek_intel_from_threat_feed.py index b2c4e6c72..f893a10d7 100755 --- a/shared/bin/zeek_intel_from_threat_feed.py +++ b/shared/bin/zeek_intel_from_threat_feed.py @@ -172,6 +172,7 @@ def main(): with malcolm_utils.temporary_filename(suffix='.txt') as tmpFileName: dlFileName = zeek_threat_feed_utils.download_to_file( infileArg, + ssl_verify=args.sslVerify, local_filename=tmpFileName, logger=logging, ) diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py index 813e9a8bb..23be91a92 100644 --- a/shared/bin/zeek_threat_feed_utils.py +++ b/shared/bin/zeek_threat_feed_utils.py @@ -36,7 +36,7 @@ import requests import urllib3 -from malcolm_utils import base64_decode_if_prefixed, LoadStrIfJson, LoadFileIfJson +from malcolm_utils import eprint, base64_decode_if_prefixed, LoadStrIfJson, LoadFileIfJson # keys for dict returned by map_stix_indicator_to_zeek for Zeek intel file fields ZEEK_INTEL_INDICATOR = 'indicator' @@ -125,11 +125,11 @@ def get_url_paths_from_response(response_text, parent_url='', ext=''): ] -def get_url_paths(url, session=None, ext='', params={}): +def get_url_paths(url, session=None, ssl_verify=False, ext='', params={}): response = ( - requests.get(url, params=params, allow_redirects=True) + requests.get(url, params=params, allow_redirects=True, verify=ssl_verify) if session is None - else session.get(url, params=params, allow_redirects=True) + else session.get(url, params=params, allow_redirects=True, verify=ssl_verify) ) if response.ok: response_text = response.text @@ -139,12 +139,12 @@ def get_url_paths(url, session=None, ext='', params={}): # download to file -def download_to_file(url, session=None, local_filename=None, chunk_bytes=4096, logger=None): +def download_to_file(url, session=None, local_filename=None, chunk_bytes=4096, ssl_verify=False, logger=None): tmpDownloadedFileSpec = local_filename if local_filename else os.path.basename(urlparse(url).path) r = ( - requests.get(url, stream=True, allow_redirects=True) + requests.get(url, stream=True, allow_redirects=True, verify=ssl_verify) if session is None - else session.get(url, stream=True, allow_redirects=True) + else session.get(url, stream=True, allow_redirects=True, verify=ssl_verify) ) with open(tmpDownloadedFileSpec, "wb") as f: for chunk in r.iter_content(chunk_size=chunk_bytes): @@ -498,63 +498,86 @@ def ProcessMISP( source: Union[Tuple[str], None] = None, url: Union[str, None] = None, ): - try: - event = MISPEvent() - event.from_dict(**toParse) - - if source is None: - source = [] + if isinstance(toParse, dict): + try: + # determine if we're processing an event or an attribute + if 'info' in toParse: + # this is an event, which contains an array of attributes + attr = None + event = MISPEvent() + event.from_dict(**toParse) + + if source is None: + source = [] + + if event.Orgc is not None: + source.append(event.Orgc.name) + + description = event.info + + if (event.Tag is not None) and (len(event.Tag) > 0): + tags = [ + x.name + for x in event.Tag + if not x.name.startswith('osint:certainty') + and not x.name.startswith('type:') + and not x.name.startswith('source:') + ] + source.extend([x.name[7:] for x in event.Tag if x.name.startswith('source:')]) + certaintyTags = [ + x.name.replace('"', '') for x in event.Tag if x.name.startswith('osint:certainty') + ] + try: + certainty = float(certaintyTags[0].split('=')[-1]) if len(certaintyTags) > 0 else None + except ValueError: + certainty = None + else: + tags = [] + certainty = None + + elif ('id' in toParse) and ('type' in toParse): + # processing a single attribute + event = None + attr = MISPAttribute() + attr.from_dict(**toParse) + # TODO: figure out what to put for description, tags, certainty, etc.? + description = '' + tags = [] + certainty = None - if event.Orgc is not None: - source.append(event.Orgc.name) + for attribute in event.attributes if event else [attr]: + # map attribute to Zeek value(s) + if ( + ((not hasattr(attribute, 'deleted')) or (not attribute.deleted)) + and ( + (self.since is None) + or (event and hasattr(event, 'timestamp') and (event.timestamp >= self.since)) + or (attribute and hasattr(attribute, 'timestamp') and attribute.timestamp >= self.since) + ) + and ( + vals := map_misp_attribute_to_zeek( + attribute=attribute, + source=source, + url=url, + description=f"{description}{'. '+attribute.comment if (hasattr(attribute, 'comment') and attribute.comment) else ''}", + tags=tags, + confidence=certainty, + logger=self.logger, + ) + ) + ): + for val in vals: + self.PrintHeader() + with self.lock: + # print the intelligence item fields according to the columns in 'fields' + print('\t'.join([val[key] for key in self.fields]), file=self.outFile) - description = event.info + except Exception as e: + if self.logger is not None: + self.logger.warning(e, exc_info=True) - if (event.Tag is not None) and (len(event.Tag) > 0): - tags = [ - x.name - for x in event.Tag - if not x.name.startswith('osint:certainty') - and not x.name.startswith('type:') - and not x.name.startswith('source:') - ] - # TODO: 'slice' object is not subscriptable - source.extend([x.name[7:] for x in event.Tag if x.name.startswith('source:')]) - certaintyTags = [x.name.replace('"', '') for x in event.Tag if x.name.startswith('osint:certainty')] - try: - certainty = float(certaintyTags[0].split('=')[-1]) if len(certaintyTags) > 0 else None - except ValueError: - certainty = None - else: - tags = [] - certainty = None - - for attribute in event.attributes: - # map event attribute to Zeek value(s) - if ( - ((not hasattr(attribute, 'deleted')) or (not attribute.deleted)) - and ((self.since is None) or (event.timestamp >= self.since) or (attribute.timestamp >= self.since)) - and ( - vals := map_misp_attribute_to_zeek( - attribute=attribute, - source=source, - url=url, - description=f"{description}{'. '+attribute.comment if attribute.comment else ''}", - tags=tags, - confidence=certainty, - logger=self.logger, - ) - ) - ): - for val in vals: - self.PrintHeader() - with self.lock: - # print the intelligence item fields according to the columns in 'fields' - print('\t'.join([val[key] for key in self.fields]), file=self.outFile) - - except Exception as e: - if self.logger is not None: - self.logger.warning(e, exc_info=True) + elif self.logger is not None: + self.logger.warning(f"Unknown MISP object format '{json.dumps(toParse)}'") def ProcessThreatInputWorker(threatInputWorkerArgs): @@ -621,6 +644,7 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): mispUrl = mispConnInfo[0] if len(mispConnInfo) >= 2: mispAuthKey = mispConnInfo[1] + reqHeaders = {'Accept': 'application/json, text/plain, text/html'} with requests.Session() as mispSession: if mispAuthKey is not None: @@ -631,20 +655,51 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): # - a directory listing *containing* a manifest.json (https://www.circl.lu/doc/misp/feed-osint/) # - a directory listing of misc. JSON files without a manifest.json # - an array of attributes returned for a request via the MISP Automation API to an /attributes endpoint - mispResponse = mispSession.get(mispUrl, verify=sslVerify) + mispResponse = mispSession.get( + mispUrl, + headers=reqHeaders, + allow_redirects=True, + verify=sslVerify, + ) mispResponse.raise_for_status() if mispJson := LoadStrIfJson(mispResponse.content): - # the contents are JSON. determine if this is a manifest or a single event - - if (len(mispJson.keys()) == 1) and ('Event' in mispJson): - # TODO: is this always the case? anything other than "Event", or multiple objects? + # the contents are JSON. determine if this is: + # - a manifest + # - an array of Attributes + # - a single Event + + if ( + isinstance(mispJson, dict) + and (len(mispJson.keys()) == 1) + and ('Event' in mispJson) + ): # this is a MISP event, process it zeekPrinter.ProcessMISP( mispJson, url=mispUrl, ) - else: + elif isinstance(mispJson, list): + # this is an array of attributes + for attr in mispJson: + if isinstance(attr, dict) and ('id' in attr) and ('type' in attr): + try: + zeekPrinter.ProcessMISP( + attr, + url=mispUrl, + ) + except Exception as e: + if logger is not None: + logger.warning( + f"{type(e).__name__} for MISP attribute '{json.dumps(attr)}': {e}" + ) + else: + if logger is not None: + logger.warning( + f"Unknown MISP attribute format '{json.dumps(attr)}'" + ) + + elif isinstance(mispJson, dict): # this is a manifest, loop over, retrieve and process the MISP events it references for uri in mispJson: try: @@ -657,7 +712,12 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): else defaultNow ) if (since is None) or (eventTime >= since): - mispObjectReponse = mispSession.get(newUrl, verify=sslVerify) + mispObjectReponse = mispSession.get( + newUrl, + headers=reqHeaders, + allow_redirects=True, + verify=sslVerify, + ) mispObjectReponse.raise_for_status() zeekPrinter.ProcessMISP( mispObjectReponse.json(), @@ -669,6 +729,9 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): f"{type(e).__name__} for MISP object at '{newUrl}': {e}" ) + else: + raise Exception(f"Unknown MISP format '{type(mispJson)}'") + else: # the contents are NOT JSON, it's probably an HTML-formatted directory listing @@ -684,7 +747,12 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): # retrieve it, then loop over it and retrieve and process the MISP events it references for url in manifestPaths: try: - mispManifestResponse = mispSession.get(url, verify=sslVerify) + mispManifestResponse = mispSession.get( + url, + headers=reqHeaders, + allow_redirects=True, + verify=sslVerify, + ) mispManifestResponse.raise_for_status() mispManifest = mispManifestResponse.json() for uri in mispManifest: @@ -699,7 +767,10 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): if (since is None) or (eventTime >= since): newUrl = f'{mispUrl.strip("/")}/{uri}.json' mispObjectReponse = mispSession.get( - newUrl, verify=sslVerify + newUrl, + headers=reqHeaders, + allow_redirects=True, + verify=sslVerify, ) mispObjectReponse.raise_for_status() zeekPrinter.ProcessMISP( @@ -720,7 +791,12 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): # just loop over, retrieve and process the .json files in this directory for url in paths: try: - mispObjectReponse = mispSession.get(url, verify=sslVerify) + mispObjectReponse = mispSession.get( + url, + headers=reqHeaders, + allow_redirects=True, + verify=sslVerify, + ) mispObjectReponse.raise_for_status() zeekPrinter.ProcessMISP( mispObjectReponse.json(), From a8a08f2e58ff13c7e9e472e5625b0187c9e1d87c Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 28 Feb 2024 11:06:28 -0700 Subject: [PATCH 49/79] work in progress for idaholab/Malcolm#336, expand MISP usage --- shared/bin/zeek_threat_feed_utils.py | 241 ++++++++++++++++----------- 1 file changed, 142 insertions(+), 99 deletions(-) diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py index 23be91a92..32915d939 100644 --- a/shared/bin/zeek_threat_feed_utils.py +++ b/shared/bin/zeek_threat_feed_utils.py @@ -12,7 +12,7 @@ from contextlib import contextmanager, nullcontext from datetime import datetime from multiprocessing import RawValue -from pymisp import MISPEvent, MISPAttribute +from pymisp import MISPEvent, MISPAttribute, PyMISP from pytz import utc as UTCTimeZone from stix2 import parse as STIXParse from stix2.exceptions import STIXError @@ -30,6 +30,7 @@ from time import sleep, mktime from typing import Tuple, Union from urllib.parse import urljoin, urlparse +from logging import DEBUG as LOGGING_DEBUG import json import os import re @@ -54,6 +55,7 @@ TAXII_INDICATOR_FILTER = {'type': 'indicator'} TAXII_PAGE_SIZE = 50 +MISP_PAGE_SIZE = 60 ZEEK_INTEL_WORKER_THREADS_DEFAULT = 2 @@ -500,77 +502,85 @@ def ProcessMISP( ): if isinstance(toParse, dict): try: + attr = None + event = None + description = '' + if source is None: + source = [] + tags = [] + certainty = None + # determine if we're processing an event or an attribute if 'info' in toParse: # this is an event, which contains an array of attributes - attr = None event = MISPEvent() event.from_dict(**toParse) - if source is None: - source = [] - - if event.Orgc is not None: - source.append(event.Orgc.name) - - description = event.info - - if (event.Tag is not None) and (len(event.Tag) > 0): - tags = [ - x.name - for x in event.Tag - if not x.name.startswith('osint:certainty') - and not x.name.startswith('type:') - and not x.name.startswith('source:') - ] - source.extend([x.name[7:] for x in event.Tag if x.name.startswith('source:')]) - certaintyTags = [ - x.name.replace('"', '') for x in event.Tag if x.name.startswith('osint:certainty') - ] - try: - certainty = float(certaintyTags[0].split('=')[-1]) if len(certaintyTags) > 0 else None - except ValueError: - certainty = None - else: - tags = [] - certainty = None - elif ('id' in toParse) and ('type' in toParse): # processing a single attribute - event = None attr = MISPAttribute() attr.from_dict(**toParse) - # TODO: figure out what to put for description, tags, certainty, etc.? - description = '' - tags = [] - certainty = None - - for attribute in event.attributes if event else [attr]: - # map attribute to Zeek value(s) - if ( - ((not hasattr(attribute, 'deleted')) or (not attribute.deleted)) - and ( - (self.since is None) - or (event and hasattr(event, 'timestamp') and (event.timestamp >= self.since)) - or (attribute and hasattr(attribute, 'timestamp') and attribute.timestamp >= self.since) - ) - and ( - vals := map_misp_attribute_to_zeek( - attribute=attribute, - source=source, - url=url, - description=f"{description}{'. '+attribute.comment if (hasattr(attribute, 'comment') and attribute.comment) else ''}", - tags=tags, - confidence=certainty, - logger=self.logger, + event = MISPEvent() + event.from_dict(**attr.Event) + + if attr or event: + if event: + # format the descriptive info for the Zeek intel item + if hasattr(event, 'Orgc') and event.Orgc: + source.append(event.Orgc.name) + elif hasattr(event, 'orgc') and event.orgc: + source.append(event.orgc.name) + + if hasattr(event, 'info') and event.info: + description = event.info + + if hasattr(event, 'Tag') and (event.Tag is not None) and (len(event.Tag) > 0): + tags = [ + x.name + for x in event.Tag + if not x.name.startswith('osint:certainty') + and not x.name.startswith('type:') + and not x.name.startswith('source:') + ] + source.extend([x.name[7:] for x in event.Tag if x.name.startswith('source:')]) + certaintyTags = [ + x.name.replace('"', '') for x in event.Tag if x.name.startswith('osint:certainty') + ] + try: + certainty = float(certaintyTags[0].split('=')[-1]) if len(certaintyTags) > 0 else None + except ValueError: + certainty = None + + # loop through and process the attribute(s) + for attribute in [attr] if attr else event.attributes: + # map attribute to Zeek value(s) + if ( + ((not hasattr(attribute, 'deleted')) or (not attribute.deleted)) + and ( + (self.since is None) + or (event and hasattr(event, 'timestamp') and (event.timestamp >= self.since)) + or (attribute and hasattr(attribute, 'timestamp') and attribute.timestamp >= self.since) ) - ) - ): - for val in vals: - self.PrintHeader() - with self.lock: - # print the intelligence item fields according to the columns in 'fields' - print('\t'.join([val[key] for key in self.fields]), file=self.outFile) + and ( + vals := map_misp_attribute_to_zeek( + attribute=attribute, + source=source, + url=url, + description=f"{description}{'. '+attribute.comment if (hasattr(attribute, 'comment') and attribute.comment) else ''}", + tags=tags, + confidence=certainty, + logger=self.logger, + ) + ) + ): + for val in vals: + self.PrintHeader() + with self.lock: + # print the intelligence item fields according to the columns in 'fields' + print('\t'.join([val[key] for key in self.fields]), file=self.outFile) + + elif self.logger is not None: + self.logger.warning(f"Unknown MISP object format '{json.dumps(toParse)}'") except Exception as e: if self.logger is not None: @@ -609,21 +619,22 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): # JSON FILE (STIX or MISP) if infileJson := LoadFileIfJson(infile): - if 'type' in infileJson and 'id' in infileJson: - # STIX input file - zeekPrinter.ProcessSTIX( - infileJson, - source=[os.path.splitext(os.path.basename(inarg))[0]], - ) - - elif (len(infileJson.keys()) == 1) and ('Event' in infileJson): - # TODO: is this always the case? anything other than "Event", or multiple objects? - # MISP input file - zeekPrinter.ProcessMISP( - infileJson, - source=[os.path.splitext(os.path.basename(inarg))[0]], - ) + if isinstance(infileJson, dict): + if 'type' in infileJson and 'id' in infileJson: + # STIX input file + zeekPrinter.ProcessSTIX( + infileJson, + source=[os.path.splitext(os.path.basename(inarg))[0]], + ) + elif (len(infileJson.keys()) == 1) and ('Event' in infileJson): + # MISP input file containing "Event" + zeekPrinter.ProcessMISP( + infileJson, + source=[os.path.splitext(os.path.basename(inarg))[0]], + ) + else: + raise Exception(f"Could not identify content in '{inarg}'") else: raise Exception(f"Could not identify content in '{inarg}'") else: @@ -644,9 +655,9 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): mispUrl = mispConnInfo[0] if len(mispConnInfo) >= 2: mispAuthKey = mispConnInfo[1] - reqHeaders = {'Accept': 'application/json, text/plain, text/html'} with requests.Session() as mispSession: + mispSession.headers.update({'Accept': 'application/json, text/plain, text/html'}) if mispAuthKey is not None: mispSession.headers.update({'Authorization': mispAuthKey}) @@ -657,7 +668,6 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): # - an array of attributes returned for a request via the MISP Automation API to an /attributes endpoint mispResponse = mispSession.get( mispUrl, - headers=reqHeaders, allow_redirects=True, verify=sslVerify, ) @@ -680,24 +690,61 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): ) elif isinstance(mispJson, list): - # this is an array of attributes - for attr in mispJson: - if isinstance(attr, dict) and ('id' in attr) and ('type' in attr): - try: - zeekPrinter.ProcessMISP( - attr, - url=mispUrl, - ) - except Exception as e: - if logger is not None: - logger.warning( - f"{type(e).__name__} for MISP attribute '{json.dumps(attr)}': {e}" - ) - else: - if logger is not None: - logger.warning( - f"Unknown MISP attribute format '{json.dumps(attr)}'" - ) + # this is an array of attributes. rather than handling it via + # additional calls with request, let's use the MISP API + # to do the searching/pulling (yeah, we're duplicating + # the effort of pulling the first page, but meh, who cares?) + if mispObject := PyMISP( + mispUrl, + mispAuthKey, + sslVerify, + debug=logger and (LOGGING_DEBUG >= logger.root.level), + ): + # search, looping over the pages MISP_PAGE_SIZE at a time + mispPage = 1 + while True: + attrResults = mispObject.search( + controller='attributes', + return_format='json', + limit=MISP_PAGE_SIZE, + page=mispPage, + type_attribute=list(MISP_ZEEK_INTEL_TYPE_MAP.keys()), + event_timestamp=since, + ) + if ( + attrResults + and isinstance(attrResults, dict) + and ('Attribute' in attrResults) + and isinstance(attrResults['Attribute'], list) + ): + mispPage += 1 + for attr in attrResults['Attribute']: + if ( + isinstance(attr, dict) + and ('id' in attr) + and ('type' in attr) + ): + try: + zeekPrinter.ProcessMISP( + attr, + url=mispUrl, + ) + except Exception as e: + if logger is not None: + logger.warning( + f"{type(e).__name__} for MISP attribute '{json.dumps(attr)}': {e}" + ) + else: + if logger is not None: + logger.warning( + f"Unknown MISP attribute format '{json.dumps(attr)}'" + ) + else: + # error or unrecognized results, set this to short circuit + attrResults = None + + if not attrResults or (len(attrResults['Attribute']) < MISP_PAGE_SIZE): + break elif isinstance(mispJson, dict): # this is a manifest, loop over, retrieve and process the MISP events it references @@ -714,7 +761,6 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): if (since is None) or (eventTime >= since): mispObjectReponse = mispSession.get( newUrl, - headers=reqHeaders, allow_redirects=True, verify=sslVerify, ) @@ -749,7 +795,6 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): try: mispManifestResponse = mispSession.get( url, - headers=reqHeaders, allow_redirects=True, verify=sslVerify, ) @@ -768,7 +813,6 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): newUrl = f'{mispUrl.strip("/")}/{uri}.json' mispObjectReponse = mispSession.get( newUrl, - headers=reqHeaders, allow_redirects=True, verify=sslVerify, ) @@ -793,7 +837,6 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): try: mispObjectReponse = mispSession.get( url, - headers=reqHeaders, allow_redirects=True, verify=sslVerify, ) From 93a00bb57f6bdeb003187e6d24796c3f94223392 Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 28 Feb 2024 12:21:13 -0700 Subject: [PATCH 50/79] work in progress for idaholab/Malcolm#336, expand MISP usage --- docs/zeek-intel.md | 4 +- shared/bin/zeek_intel_from_threat_feed.py | 4 +- shared/bin/zeek_threat_feed_utils.py | 164 ++++++++++++++-------- 3 files changed, 109 insertions(+), 63 deletions(-) diff --git a/docs/zeek-intel.md b/docs/zeek-intel.md index 6b0704ed6..1d853f66a 100644 --- a/docs/zeek-intel.md +++ b/docs/zeek-intel.md @@ -56,6 +56,7 @@ For example: misp|https://example.com/data/feed-osint/manifest.json|df97338db644c64fbfd90f3e03ba8870 misp|https://example.com/doc/misp/| misp|https://example.com/attributes|a943f5ff506ee6198e996333e0b672b1 +misp|https://example.com/events|a943f5ff506ee6198e996333e0b672b1 … ``` @@ -66,6 +67,7 @@ Upon Malcolm connects to the URLs for the MISP feeds in `.misp_input.txt`, it wi * a manifest JSON file * a directory listing containing a file named `manifest.json` * a directory listing of JSON files without a `manifest.json` file -* an array of [attributes](https://www.misp-project.org/openapi/#tag/Attributes) returned for a request via the [MISP Automation API](https://www.misp-project.org/openapi/) made to a MISP platform's `/attributes` endpoint +* a list of [Events](https://www.misp-project.org/openapi/#tag/Events) returned for a request via the [MISP Automation API](https://www.misp-project.org/openapi/) made to a MISP platform's [`/events` endpoint](https://www.misp-project.org/openapi/#tag/Events/operation/restSearchEvents) +* a list of [Attributes](https://www.misp-project.org/openapi/#tag/Attributes) returned for a request via the [MISP Automation API](https://www.misp-project.org/openapi/) made to a MISP platform's [`/attributes` endpoint](https://www.misp-project.org/openapi/#tag/Attributes/operation/restSearchAttributes) Note that only a subset of MISP [attribute types](https://www.misp-project.org/datamodels/#attribute-categories-vs-types) can be expressed with the Zeek intelligence [indicator types](https://docs.zeek.org/en/master/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Type). MISP attributes with other types will be silently ignored. \ No newline at end of file diff --git a/shared/bin/zeek_intel_from_threat_feed.py b/shared/bin/zeek_intel_from_threat_feed.py index f893a10d7..af24b72e0 100755 --- a/shared/bin/zeek_intel_from_threat_feed.py +++ b/shared/bin/zeek_intel_from_threat_feed.py @@ -143,8 +143,8 @@ def main(): logging.info(os.path.join(script_path, script_name)) logging.info("Arguments: {}".format(sys.argv[1:])) logging.info("Arguments: {}".format(args)) - if args.verbose > logging.DEBUG: - sys.tracebacklimit = 0 + # if args.verbose > logging.DEBUG: + # sys.tracebacklimit = 0 if args.input is None: args.input = [] diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py index 32915d939..e79bd26c4 100644 --- a/shared/bin/zeek_threat_feed_utils.py +++ b/shared/bin/zeek_threat_feed_utils.py @@ -55,7 +55,8 @@ TAXII_INDICATOR_FILTER = {'type': 'indicator'} TAXII_PAGE_SIZE = 50 -MISP_PAGE_SIZE = 60 +MISP_PAGE_SIZE_ATTRIBUTES = 60 +MISP_PAGE_SIZE_EVENTS = 10 ZEEK_INTEL_WORKER_THREADS_DEFAULT = 2 @@ -512,7 +513,7 @@ def ProcessMISP( # determine if we're processing an event or an attribute if 'info' in toParse: - # this is an event, which contains an array of attributes + # this is an event, which may contain an array of attributes event = MISPEvent() event.from_dict(**toParse) @@ -580,14 +581,14 @@ def ProcessMISP( print('\t'.join([val[key] for key in self.fields]), file=self.outFile) elif self.logger is not None: - self.logger.warning(f"Unknown MISP object format '{json.dumps(toParse)}'") + self.logger.warning("Unknown MISP object format (could not determine Attribute vs. Event)") except Exception as e: if self.logger is not None: self.logger.warning(e, exc_info=True) elif self.logger is not None: - self.logger.warning(f"Unknown MISP object format '{json.dumps(toParse)}'") + self.logger.warning(f"Unknown MISP object format ('{type(toParse)}')") def ProcessThreatInputWorker(threatInputWorkerArgs): @@ -603,7 +604,7 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): with workerThreadCount as workerId: if logger is not None: - logger.debug(f"[{workerId}]:started") + logger.debug(f"[{workerId}]: started") # the queue was fully populated before we started, so we can run until there are no more elements while len(inputQueue) > 0: @@ -665,7 +666,8 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): # - a manifest JSON (https://www.circl.lu/doc/misp/feed-osint/manifest.json) # - a directory listing *containing* a manifest.json (https://www.circl.lu/doc/misp/feed-osint/) # - a directory listing of misc. JSON files without a manifest.json - # - an array of attributes returned for a request via the MISP Automation API to an /attributes endpoint + # - an array of Attributes returned for a request via the MISP Automation API to an /attributes endpoint + # - an array of Events returned for a request via the MISP Automation API to an /events endpoint mispResponse = mispSession.get( mispUrl, allow_redirects=True, @@ -674,77 +676,115 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): mispResponse.raise_for_status() if mispJson := LoadStrIfJson(mispResponse.content): # the contents are JSON. determine if this is: - # - a manifest - # - an array of Attributes # - a single Event + # - an array of Events + # - an array of Attributes + # - a manifest if ( isinstance(mispJson, dict) and (len(mispJson.keys()) == 1) and ('Event' in mispJson) ): - # this is a MISP event, process it + # this is a single MISP Event, process it zeekPrinter.ProcessMISP( mispJson, url=mispUrl, ) - elif isinstance(mispJson, list): - # this is an array of attributes. rather than handling it via - # additional calls with request, let's use the MISP API - # to do the searching/pulling (yeah, we're duplicating - # the effort of pulling the first page, but meh, who cares?) - if mispObject := PyMISP( - mispUrl, - mispAuthKey, - sslVerify, - debug=logger and (LOGGING_DEBUG >= logger.root.level), + elif isinstance(mispJson, list) and (len(mispJson) > 0): + # are these Attributes or Events? + if ( + isinstance(mispJson[0], dict) + and ('id' in mispJson[0]) + and ('type' in mispJson[0]) ): - # search, looping over the pages MISP_PAGE_SIZE at a time - mispPage = 1 - while True: - attrResults = mispObject.search( - controller='attributes', - return_format='json', - limit=MISP_PAGE_SIZE, - page=mispPage, - type_attribute=list(MISP_ZEEK_INTEL_TYPE_MAP.keys()), - event_timestamp=since, - ) - if ( - attrResults - and isinstance(attrResults, dict) - and ('Attribute' in attrResults) - and isinstance(attrResults['Attribute'], list) - ): + controllerType = 'attributes' + resultKey = 'Attribute' + pageSize = MISP_PAGE_SIZE_ATTRIBUTES + elif isinstance(mispJson[0], dict) and ('info' in mispJson[0]): + controllerType = 'events' + resultKey = 'Event' + pageSize = MISP_PAGE_SIZE_EVENTS + else: + controllerType = None + resultKey = None + pageSize = None + + if controllerType: + # this is an array of either Attributes or Events. + # rather than handling it via additional calls with request, + # let's use the MISP API to do the searching/pulling + # (yeah, we're duplicating the effort of pulling the + # first page, but meh, who cares?) + if mispObject := PyMISP( + mispUrl, + mispAuthKey, + sslVerify, + debug=logger and (LOGGING_DEBUG >= logger.root.level), + ): + # search, looping over the pages pageSize at a time + mispPage = 0 + while True: mispPage += 1 - for attr in attrResults['Attribute']: - if ( - isinstance(attr, dict) - and ('id' in attr) - and ('type' in attr) - ): + resultCount = 0 + mispResults = mispObject.search( + controller=controllerType, + return_format='json', + limit=pageSize, + page=mispPage, + type_attribute=list(MISP_ZEEK_INTEL_TYPE_MAP.keys()), + timestamp=since, + ) + if ( + mispResults + and isinstance(mispResults, dict) + and (resultKey in mispResults) + ): + # Attributes results + resultCount = len(mispResults[resultKey]) + for item in mispResults[resultKey]: try: zeekPrinter.ProcessMISP( - attr, + item, url=mispUrl, ) except Exception as e: if logger is not None: logger.warning( - f"{type(e).__name__} for MISP attribute '{json.dumps(attr)}': {e}" + f"[{workerId}]: {type(e).__name__} for MISP {resultKey}: {e}" ) - else: - if logger is not None: - logger.warning( - f"Unknown MISP attribute format '{json.dumps(attr)}'" - ) - else: - # error or unrecognized results, set this to short circuit - attrResults = None - if not attrResults or (len(attrResults['Attribute']) < MISP_PAGE_SIZE): - break + elif mispResults and isinstance(mispResults, list): + # Events results + resultCount = len(mispResults) + for item in mispResults: + if item and isinstance(item, dict) and (resultKey in item): + try: + zeekPrinter.ProcessMISP( + item[resultKey], + url=mispUrl, + ) + except Exception as e: + if logger is not None: + logger.warning( + f"[{workerId}]: {type(e).__name__} for MISP {resultKey}: {e}" + ) + + else: + # error or unrecognized results, set this to short circuit + resultCount = 0 + + if logger is not None: + logger.debug( + f"[{workerId}]: MISP search page {mispPage} returned {resultCount}" + ) + if not mispResults or (resultCount < pageSize): + break + + else: + # not an Event or an Attribute? what the heck are we even doing? + raise Exception(f"Unknown MISP object '{json.dumps(mispJson)}'") elif isinstance(mispJson, dict): # this is a manifest, loop over, retrieve and process the MISP events it references @@ -772,7 +812,7 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): except Exception as e: if logger is not None: logger.warning( - f"{type(e).__name__} for MISP object at '{newUrl}': {e}" + f"[{workerId}]: {type(e).__name__} for MISP object at '{newUrl}': {e}" ) else: @@ -824,11 +864,13 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): except Exception as e: if logger is not None: logger.warning( - f"{type(e).__name__} for MISP object at '{mispUrl}/{uri}.json': {e}" + f"[{workerId}]: {type(e).__name__} for MISP object at '{mispUrl}/{uri}.json': {e}" ) except Exception as e: if logger is not None: - logger.warning(f"{type(e).__name__} for manifest at '{url}': {e}") + logger.warning( + f"[{workerId}]: {type(e).__name__} for manifest at '{url}': {e}" + ) else: # the manifest.json does not exist! @@ -848,7 +890,7 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): except Exception as e: if logger is not None: logger.warning( - f"{type(e).__name__} for MISP object at '{url}': {e}" + f"[{workerId}]: {type(e).__name__} for MISP object at '{url}': {e}" ) elif inarg.lower().startswith('taxii'): @@ -936,11 +978,13 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): except Exception as e: if logger is not None: - logger.warning(f"{type(e).__name__} for object of collection '{title}': {e}") + logger.warning( + f"[{workerId}]: {type(e).__name__} for object of collection '{title}': {e}" + ) except Exception as e: if logger is not None: - logger.warning(f"{type(e).__name__} for '{inarg}': {e}") + logger.warning(f"[{workerId}]: {type(e).__name__} for '{inarg}': {e}") if logger is not None: logger.debug(f"[{workerId}]: finished") From 2f5d07c7d8ba8dc7ec1a9ed746d8c49c3fd11b5d Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 28 Feb 2024 13:07:42 -0700 Subject: [PATCH 51/79] work in progress for idaholab/Malcolm#336, expand MISP usage --- shared/bin/zeek_intel_from_threat_feed.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/bin/zeek_intel_from_threat_feed.py b/shared/bin/zeek_intel_from_threat_feed.py index af24b72e0..f893a10d7 100755 --- a/shared/bin/zeek_intel_from_threat_feed.py +++ b/shared/bin/zeek_intel_from_threat_feed.py @@ -143,8 +143,8 @@ def main(): logging.info(os.path.join(script_path, script_name)) logging.info("Arguments: {}".format(sys.argv[1:])) logging.info("Arguments: {}".format(args)) - # if args.verbose > logging.DEBUG: - # sys.tracebacklimit = 0 + if args.verbose > logging.DEBUG: + sys.tracebacklimit = 0 if args.input is None: args.input = [] From 80a4f86a91419bf32acbcb33eaedb4da82da8050 Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 28 Feb 2024 15:40:22 -0700 Subject: [PATCH 52/79] work in progress for idaholab/Malcolm#336, expand MISP usage --- shared/bin/zeek_threat_feed_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py index e79bd26c4..229fd0449 100644 --- a/shared/bin/zeek_threat_feed_utils.py +++ b/shared/bin/zeek_threat_feed_utils.py @@ -55,7 +55,7 @@ TAXII_INDICATOR_FILTER = {'type': 'indicator'} TAXII_PAGE_SIZE = 50 -MISP_PAGE_SIZE_ATTRIBUTES = 60 +MISP_PAGE_SIZE_ATTRIBUTES = 500 MISP_PAGE_SIZE_EVENTS = 10 ZEEK_INTEL_WORKER_THREADS_DEFAULT = 2 From 33ef67efd522a19f71bc37226505c1f55ea33148 Mon Sep 17 00:00:00 2001 From: Melanie Pierce Date: Thu, 29 Feb 2024 08:32:04 -0700 Subject: [PATCH 53/79] Fix missing ")" --- docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index 6883a6d8f..71196f1cb 100644 --- a/docs/README.md +++ b/docs/README.md @@ -85,7 +85,7 @@ Malcolm can also easily be deployed locally on an ordinary consumer workstation + [Other Customizations](custom-rules.md#Other) - [Automatic file extraction and scanning](file-scanning.md#ZeekFileExtraction) + [User interface](file-scanning.md#ZeekFileExtractionUI) - - [Index management](index-management.md# + - [Index management](index-management.md#) + [OpenSearch index management](index-management.md#OpenSearchIndexManagement) + [Using ILM/ISM with Arkime](index-management.md#ArkimeIndexPolicies) - [Event severity scoring](severity.md#Severity) From a69381cc279df09640cd14c5c6a5a5658374020c Mon Sep 17 00:00:00 2001 From: Melanie Pierce Date: Wed, 28 Feb 2024 17:12:17 -0700 Subject: [PATCH 54/79] Add documentation to log out of Malcolm --- docs/README.md | 1 + docs/authsetup.md | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/docs/README.md b/docs/README.md index 71196f1cb..2d2f14741 100644 --- a/docs/README.md +++ b/docs/README.md @@ -33,6 +33,7 @@ Malcolm can also easily be deployed locally on an ordinary consumer workstation * [LDAP connection security](authsetup.md#AuthLDAPSecurity) + [TLS certificates](authsetup.md#TLSCerts) + [Command-line arguments](authsetup.md#CommandLineConfig) + - [Log Out of Malcolm](authsetup.md#LoggingOut) - [Platform-specific Configuration](host-config.md#HostSystemConfig) + [Linux host system configuration](host-config-linux.md#HostSystemConfigLinux) + [macOS host system configuration](host-config-macos.md#HostSystemConfigMac) diff --git a/docs/authsetup.md b/docs/authsetup.md index 49b707643..19100a8cc 100644 --- a/docs/authsetup.md +++ b/docs/authsetup.md @@ -6,6 +6,7 @@ + [LDAP connection security](#AuthLDAPSecurity) - [TLS certificates](#TLSCerts) - [Command-line arguments](#CommandLineConfig) +* [Log Out of Malcolm](#LoggingOut) Malcolm requires authentication to access the [user interface](quickstart.md#UserInterfaceURLs). [Nginx](https://nginx.org/) can authenticate users with either local TLS-encrypted HTTP basic authentication or a remote Lightweight Directory Access Protocol (LDAP) authentication server. @@ -146,3 +147,9 @@ Authentication Setup: ``` In order to avoid specifying passwords as command line arguments, when setting the administrator password noninteractively the user must instead provide hashes generated by [`openssl -passwd -1`](https://www.openssl.org/docs/man1.1.1/man1/openssl-passwd.html) and [`htpasswd -n -B username`](https://httpd.apache.org/docs/2.4/programs/htpasswd.html). + +# Log Out of Malcolm + +Since Malcolm's nginx reverse proxy uses HTTP Basic Authentication, there is no "Logout" button. Users should exit their browser window to log out of Malcom. On the next browser restart, Malcolm will prompt the user for credentials. + +Another way to log out of Malcolm is for a user to manually clear their browser's active sessions. From d5bf5d0b1369399a381371b437d27814fcc45071 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 29 Feb 2024 09:02:17 -0700 Subject: [PATCH 55/79] update the cryptography library to address CVE-2024-26130 (https://github.com/mmguero-dev/Malcolm/security/code-scanning/290), will no longer be necessary when idaholab/Malcolm#385 is done --- Dockerfiles/netbox.Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index c513fe26a..986528f9b 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -81,6 +81,8 @@ RUN apt-get -q update && \ python-magic \ python-slugify \ randomcolor && \ + "${NETBOX_PATH}/venv/bin/python" -m pip install --break-system-packages --no-compile --no-cache-dir --upgrade \ + cryptography && \ cd "${NETBOX_PATH}" && \ bash -c 'for i in /tmp/netbox-patches/*; do patch -p 1 -r - --no-backup-if-mismatch < $i || true; done' && \ curl -fsSLO "${SUPERCRONIC_URL}" && \ From 8fafa620955b8d4e1eb56d2502d2cf8c552fe294 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 29 Feb 2024 09:16:01 -0700 Subject: [PATCH 56/79] update a few libraries/components to address CVEs --- Dockerfiles/netbox.Dockerfile | 6 ++++-- Dockerfiles/suricata.Dockerfile | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index 986528f9b..f306da8a4 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -32,7 +32,7 @@ ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV NETBOX_INITIALIZERS_VERSION "ebf1f76" -ENV YQ_VERSION "4.33.3" +ENV YQ_VERSION "4.42.1" ENV YQ_URL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64" ENV NETBOX_DEVICETYPE_LIBRARY_IMPORT_URL "https://codeload.github.com/netbox-community/Device-Type-Library-Import/tar.gz/develop" @@ -82,7 +82,9 @@ RUN apt-get -q update && \ python-slugify \ randomcolor && \ "${NETBOX_PATH}/venv/bin/python" -m pip install --break-system-packages --no-compile --no-cache-dir --upgrade \ - cryptography && \ + cryptography \ + GitPython \ + pillow && \ cd "${NETBOX_PATH}" && \ bash -c 'for i in /tmp/netbox-patches/*; do patch -p 1 -r - --no-backup-if-mismatch < $i || true; done' && \ curl -fsSLO "${SUPERCRONIC_URL}" && \ diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index e75aa2d30..6defb77fd 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -36,7 +36,7 @@ ENV SUPERCRONIC "supercronic-linux-amd64" ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b" ENV SUPERCRONIC_CRONTAB "/etc/crontab" -ENV YQ_VERSION "4.33.3" +ENV YQ_VERSION "4.42.1" ENV YQ_URL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64" ENV SURICATA_CONFIG_DIR /etc/suricata From 866fdeec78cdea20087f9a38f23e6715b1e08bcd Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 29 Feb 2024 09:28:23 -0700 Subject: [PATCH 57/79] update a few libraries/components to address CVEs --- Dockerfiles/netbox.Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index f306da8a4..d687f92be 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -81,10 +81,6 @@ RUN apt-get -q update && \ python-magic \ python-slugify \ randomcolor && \ - "${NETBOX_PATH}/venv/bin/python" -m pip install --break-system-packages --no-compile --no-cache-dir --upgrade \ - cryptography \ - GitPython \ - pillow && \ cd "${NETBOX_PATH}" && \ bash -c 'for i in /tmp/netbox-patches/*; do patch -p 1 -r - --no-backup-if-mismatch < $i || true; done' && \ curl -fsSLO "${SUPERCRONIC_URL}" && \ @@ -110,6 +106,10 @@ RUN apt-get -q update && \ mkdir -p ./repo && \ curl -sSL "${NETBOX_DEVICETYPE_LIBRARY_URL}" | tar xzf - -C ./repo --strip-components 1 && \ rm -rf ./repo/device-types/WatchGuard && \ + "${NETBOX_PATH}/venv/bin/python" -m pip install --break-system-packages --no-compile --no-cache-dir --upgrade \ + cryptography \ + GitPython \ + pillow && \ mkdir -p "${NETBOX_PATH}/netbox/${BASE_PATH}" && \ mv "${NETBOX_PATH}/netbox/static" "${NETBOX_PATH}/netbox/${BASE_PATH}/static" && \ jq '. += { "settings": { "http": { "discard_unsafe_fields": false } } }' /etc/unit/nginx-unit.json | jq 'del(.listeners."[::]:8080")' | jq 'del(.listeners."[::]:8081")' | jq ".routes.main[0].match.uri = \"/${BASE_PATH}/static/*\"" > /etc/unit/nginx-unit-new.json && \ From 0c26007e32e536577a0216d3fdcaea6458654d0d Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 29 Feb 2024 09:44:34 -0700 Subject: [PATCH 58/79] update a few libraries/components to address CVEs --- Dockerfiles/netbox.Dockerfile | 3 ++ .../patch/nodemailer_6.9.9_package_json.patch | 38 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 arkime/patch/nodemailer_6.9.9_package_json.patch diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index d687f92be..e3cce9258 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -109,6 +109,9 @@ RUN apt-get -q update && \ "${NETBOX_PATH}/venv/bin/python" -m pip install --break-system-packages --no-compile --no-cache-dir --upgrade \ cryptography \ GitPython \ + Jinja2 \ + "Django>=4.2.10,<5" \ + paramiko \ pillow && \ mkdir -p "${NETBOX_PATH}/netbox/${BASE_PATH}" && \ mv "${NETBOX_PATH}/netbox/static" "${NETBOX_PATH}/netbox/${BASE_PATH}/static" && \ diff --git a/arkime/patch/nodemailer_6.9.9_package_json.patch b/arkime/patch/nodemailer_6.9.9_package_json.patch new file mode 100644 index 000000000..d8b0633e0 --- /dev/null +++ b/arkime/patch/nodemailer_6.9.9_package_json.patch @@ -0,0 +1,38 @@ +diff --git a/package-lock.json b/package-lock.json +index a1298740..858b0b13 100644 +--- a/package-lock.json ++++ b/package-lock.json +@@ -6817,7 +6817,7 @@ + "form-data": "4.0.0", + "https-proxy-agent": "5.0.0", + "node-fetch": "2.6.7", + "nodemailer": "6.[-6-]{+9+}.[-2-]{+9+}", + "web-push": "3.4.5", + "winston": "3.3.3" + } +@@ -16961,8 +16961,8 @@ + "dev": true + }, + "node_modules/nodemailer": { + "version": "6.[-6-]{+9+}.[-2-]{+9+}", + "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.[-6-]{+9+}.[-2-]{+9+}.tgz", + "integrity": "sha512-YSzu7TLbI+bsjCis/TZlAXBoM4y93HhlIgo0P5oiA2ua9Z4k+E2Fod//ybIzdJxOlXGRcHIh/WaeCBehvxZb/Q==", + "engines": { + "node": ">=6.0.0" +@@ -28601,7 +28601,7 @@ + "form-data": "4.0.0", + "https-proxy-agent": "5.0.0", + "node-fetch": "2.6.7", + "nodemailer": "6.[-6-]{+9+}.[-2-]{+9+}", + "web-push": "3.4.5", + "winston": "3.3.3" + } +@@ -36321,8 +36321,8 @@ + "dev": true + }, + "nodemailer": { + "version": "6.[-6-]{+9+}.[-2-]{+9+}", + "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.[-6-]{+9+}.[-2-]{+9+}.tgz", + "integrity": "sha512-YSzu7TLbI+bsjCis/TZlAXBoM4y93HhlIgo0P5oiA2ua9Z4k+E2Fod//ybIzdJxOlXGRcHIh/WaeCBehvxZb/Q==" + }, + "nodemon": { From f2c45d86ce7abf6f8e42c91eac596f1548e8aaca Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 29 Feb 2024 09:58:38 -0700 Subject: [PATCH 59/79] remove patch --- .../patch/nodemailer_6.9.9_package_json.patch | 38 ------------------- 1 file changed, 38 deletions(-) delete mode 100644 arkime/patch/nodemailer_6.9.9_package_json.patch diff --git a/arkime/patch/nodemailer_6.9.9_package_json.patch b/arkime/patch/nodemailer_6.9.9_package_json.patch deleted file mode 100644 index d8b0633e0..000000000 --- a/arkime/patch/nodemailer_6.9.9_package_json.patch +++ /dev/null @@ -1,38 +0,0 @@ -diff --git a/package-lock.json b/package-lock.json -index a1298740..858b0b13 100644 ---- a/package-lock.json -+++ b/package-lock.json -@@ -6817,7 +6817,7 @@ - "form-data": "4.0.0", - "https-proxy-agent": "5.0.0", - "node-fetch": "2.6.7", - "nodemailer": "6.[-6-]{+9+}.[-2-]{+9+}", - "web-push": "3.4.5", - "winston": "3.3.3" - } -@@ -16961,8 +16961,8 @@ - "dev": true - }, - "node_modules/nodemailer": { - "version": "6.[-6-]{+9+}.[-2-]{+9+}", - "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.[-6-]{+9+}.[-2-]{+9+}.tgz", - "integrity": "sha512-YSzu7TLbI+bsjCis/TZlAXBoM4y93HhlIgo0P5oiA2ua9Z4k+E2Fod//ybIzdJxOlXGRcHIh/WaeCBehvxZb/Q==", - "engines": { - "node": ">=6.0.0" -@@ -28601,7 +28601,7 @@ - "form-data": "4.0.0", - "https-proxy-agent": "5.0.0", - "node-fetch": "2.6.7", - "nodemailer": "6.[-6-]{+9+}.[-2-]{+9+}", - "web-push": "3.4.5", - "winston": "3.3.3" - } -@@ -36321,8 +36321,8 @@ - "dev": true - }, - "nodemailer": { - "version": "6.[-6-]{+9+}.[-2-]{+9+}", - "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.[-6-]{+9+}.[-2-]{+9+}.tgz", - "integrity": "sha512-YSzu7TLbI+bsjCis/TZlAXBoM4y93HhlIgo0P5oiA2ua9Z4k+E2Fod//ybIzdJxOlXGRcHIh/WaeCBehvxZb/Q==" - }, - "nodemon": { From f80a0bc1df3fc366e9d7a07173e4b21612457d54 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 29 Feb 2024 10:59:36 -0700 Subject: [PATCH 60/79] fix a few issues with the install.py script when installing from GitHub releases, idaholab/Malcolm#416 --- scripts/install.py | 37 +++++++++++++++++++++---------------- scripts/malcolm_common.py | 8 +++++--- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/scripts/install.py b/scripts/install.py index 18ad5a85a..c7e2ddad6 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -420,10 +420,6 @@ def install_malcolm_files(self, malcolm_install_file, default_config_dir): if os.path.isfile(os.path.join(installPath, "docker-compose.yml")): eprint(f"Malcolm runtime files extracted to {installPath}") result = True - with open(os.path.join(installPath, "install_source.txt"), 'w') as f: - f.write( - f'{os.path.basename(malcolm_install_file)} (installed {str(datetime.datetime.now())})\n' - ) else: eprint(f"Malcolm install file extracted to {installPath}, but missing runtime files?") @@ -711,9 +707,11 @@ def tweak_malcolm_runtime(self, malcolm_install_path): ) while ( not InstallerYesOrNo( - f'Setting {osMemory} for OpenSearch and {lsMemory} for Logstash. Is this OK?' - if opensearchPrimaryMode == DatabaseMode.OpenSearchLocal - else f'Setting {lsMemory} for Logstash. Is this OK?', + ( + f'Setting {osMemory} for OpenSearch and {lsMemory} for Logstash. Is this OK?' + if opensearchPrimaryMode == DatabaseMode.OpenSearchLocal + else f'Setting {lsMemory} for Logstash. Is this OK?' + ), default=True, ) and loopBreaker.increment() @@ -976,6 +974,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): # make sure paths specified (and their necessary children) exist for pathToCreate in ( + malcolm_install_path, indexDirFull, indexSnapshotDirFull, os.path.join(pcapDirFull, 'arkime-live'), @@ -1075,9 +1074,11 @@ def tweak_malcolm_runtime(self, malcolm_install_path): break if InstallerYesOrNo( - 'Should Malcolm delete the oldest database indices and/or PCAP files based on available storage?' - if ((opensearchPrimaryMode == DatabaseMode.OpenSearchLocal) and (malcolmProfile == PROFILE_MALCOLM)) - else 'Should Arkime delete PCAP files based on available storage (see https://arkime.com/faq#pcap-deletion)?', + ( + 'Should Malcolm delete the oldest database indices and/or PCAP files based on available storage?' + if ((opensearchPrimaryMode == DatabaseMode.OpenSearchLocal) and (malcolmProfile == PROFILE_MALCOLM)) + else 'Should Arkime delete PCAP files based on available storage (see https://arkime.com/faq#pcap-deletion)?' + ), default=args.arkimeManagePCAP or bool(args.indexPruneSizeLimit), ): # delete oldest indexes based on index pattern size @@ -1292,9 +1293,11 @@ def tweak_malcolm_runtime(self, malcolm_install_path): ( x, '', - x == filePreserveModeDefault - if filePreserveModeDefault - else allowedFilePreserveModes[0], + ( + x == filePreserveModeDefault + if filePreserveModeDefault + else allowedFilePreserveModes[0] + ), ) for x in allowedFilePreserveModes ], @@ -1307,9 +1310,11 @@ def tweak_malcolm_runtime(self, malcolm_install_path): 'ZIP downloaded preserved files?', default=args.fileCarveHttpServerZip ) fileCarveHttpServeEncryptKey = InstallerAskForString( - 'Enter ZIP archive password for downloaded preserved files (or leave blank for unprotected)' - if fileCarveHttpServerZip - else 'Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted)', + ( + 'Enter ZIP archive password for downloaded preserved files (or leave blank for unprotected)' + if fileCarveHttpServerZip + else 'Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted)' + ), default=args.fileCarveHttpServeEncryptKey, ) if fileCarveMode is not None: diff --git a/scripts/malcolm_common.py b/scripts/malcolm_common.py index d8e95ded3..dbb31f605 100644 --- a/scripts/malcolm_common.py +++ b/scripts/malcolm_common.py @@ -259,9 +259,11 @@ def AskForString( elif (uiMode & UserInterfaceMode.InteractionDialog) and (MainDialog is not None): code, reply = MainDialog.inputbox( question, - init=default - if (default is not None) and (defaultBehavior & UserInputDefaultsBehavior.DefaultsPrompt) - else "", + init=( + default + if (default is not None) and (defaultBehavior & UserInputDefaultsBehavior.DefaultsPrompt) + else "" + ), ) if (code == Dialog.CANCEL) or (code == Dialog.ESC): raise RuntimeError("Operation cancelled") From 35e1aac36e6612e3da1202715bb56dedbab71220 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 29 Feb 2024 11:07:05 -0700 Subject: [PATCH 61/79] KEV detections for Unitronics VisiLogic CVE-2023-6448 idaholab/Malcolm#394 --- .../OT/malcolm/CVE-2023-6448_Unitronics_VisiLogic.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 suricata/rules-default/OT/malcolm/CVE-2023-6448_Unitronics_VisiLogic.txt diff --git a/suricata/rules-default/OT/malcolm/CVE-2023-6448_Unitronics_VisiLogic.txt b/suricata/rules-default/OT/malcolm/CVE-2023-6448_Unitronics_VisiLogic.txt new file mode 100644 index 000000000..73744defe --- /dev/null +++ b/suricata/rules-default/OT/malcolm/CVE-2023-6448_Unitronics_VisiLogic.txt @@ -0,0 +1,3 @@ +alert tcp any any -> any 20256 (msg:"Suspected Unitronics Default admin Credential Use"; content:"admin"; content:"1111"; sid:1001020;) +alert tcp any any -> any 20256 (msg:"Suspected Unitronics Default user Credential Use"; content:"user"; content:"1111"; sid:1001021;) +alert tcp any any -> any 20256 (msg:"Suspected Unitronics Default operator Credential Use"; content:"operator"; content:"1111"; sid:1001022;) From 8fc4bde3b362d733aa2ef5070cfa218cde0e93c7 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 29 Feb 2024 12:03:13 -0700 Subject: [PATCH 62/79] rename suricata rules file --- ...ics_VisiLogic.txt => CVE-2023-6448_Unitronics_VisiLogic.rules} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename suricata/rules-default/OT/malcolm/{CVE-2023-6448_Unitronics_VisiLogic.txt => CVE-2023-6448_Unitronics_VisiLogic.rules} (100%) diff --git a/suricata/rules-default/OT/malcolm/CVE-2023-6448_Unitronics_VisiLogic.txt b/suricata/rules-default/OT/malcolm/CVE-2023-6448_Unitronics_VisiLogic.rules similarity index 100% rename from suricata/rules-default/OT/malcolm/CVE-2023-6448_Unitronics_VisiLogic.txt rename to suricata/rules-default/OT/malcolm/CVE-2023-6448_Unitronics_VisiLogic.rules From c83523311ebaec702898825821213a9574ae548d Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 29 Feb 2024 14:24:57 -0700 Subject: [PATCH 63/79] minor tweaks as preparing for v24.03.0 --- Dockerfiles/nginx.Dockerfile | 7 ++--- .../0d4955f0-eb25-11ec-a6d4-b3526526c2c7.json | 22 ++++++++-------- .../dashboards/beats/Filebeat-nginx-logs.json | 4 +-- .../beats/Filebeat-nginx-overview.json | 5 ++-- .../beats/Metricbeat-host-overview.json | 4 +-- .../beats/Metricbeat-system-overview.json | 4 +-- .../f6600310-9943-11ee-a029-e973f4774355.json | 26 +++++++++---------- dashboards/opensearch_dashboards.yml | 2 +- logstash/pipelines/zeek/11_zeek_parse.conf | 23 +++++++++++++--- 9 files changed, 57 insertions(+), 40 deletions(-) diff --git a/Dockerfiles/nginx.Dockerfile b/Dockerfiles/nginx.Dockerfile index ecb840a21..4dd01ec0a 100644 --- a/Dockerfiles/nginx.Dockerfile +++ b/Dockerfiles/nginx.Dockerfile @@ -244,12 +244,13 @@ ADD nginx/scripts /usr/local/bin/ ADD nginx/*.conf /etc/nginx/ ADD nginx/templates /etc/nginx/templates/ ADD nginx/supervisord.conf /etc/ +COPY --chmod=644 docs/images/favicon/*.png /usr/share/nginx/html/assets/img/ +COPY --chmod=644 docs/images/icon/*.png /usr/share/nginx/html/assets/img/ +COPY --chmod=644 docs/images/icon/*.svg /usr/share/nginx/html/assets/img/ COPY --chmod=644 docs/images/icon/favicon.ico /usr/share/nginx/html/assets/favicon.ico COPY --chmod=644 docs/images/icon/favicon.ico /usr/share/nginx/html/favicon.ico -COPY --chmod=644 docs/images/icon/*.svg /usr/share/nginx/html/assets/img/ -COPY --chmod=644 docs/images/icon/*.png /usr/share/nginx/html/assets/img/ -COPY --chmod=644 docs/images/logo/*.svg /usr/share/nginx/html/assets/img/ COPY --chmod=644 docs/images/logo/*.png /usr/share/nginx/html/assets/img/ +COPY --chmod=644 docs/images/logo/*.svg /usr/share/nginx/html/assets/img/ VOLUME ["/etc/nginx/certs", "/etc/nginx/dhparam"] diff --git a/dashboards/dashboards/beats/0d4955f0-eb25-11ec-a6d4-b3526526c2c7.json b/dashboards/dashboards/beats/0d4955f0-eb25-11ec-a6d4-b3526526c2c7.json index acd0455b2..2903b4b40 100644 --- a/dashboards/dashboards/beats/0d4955f0-eb25-11ec-a6d4-b3526526c2c7.json +++ b/dashboards/dashboards/beats/0d4955f0-eb25-11ec-a6d4-b3526526c2c7.json @@ -10,9 +10,9 @@ "updated_at": "2022-06-13T14:30:49.985Z", "version": "WzkxMSwyXQ==", "attributes": { - "title": "Malcolm Sensor Temperature", + "title": "Hardware Temperature", "hits": 0, - "description": "Malcolm sensor hardware component temperature summary", + "description": "Hardware component temperature summary, including from Malcolm sensors and aggregators", "panelsJSON": "[{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":13,\"h\":9,\"i\":\"0a9ba334-b226-4245-b00e-ad8fcabf0e00\"},\"panelIndex\":\"0a9ba334-b226-4245-b00e-ad8fcabf0e00\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_0\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":13,\"y\":0,\"w\":35,\"h\":18,\"i\":\"47ef4d93-d7d5-41c3-a3f4-d3b543ad22bc\"},\"panelIndex\":\"47ef4d93-d7d5-41c3-a3f4-d3b543ad22bc\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":9,\"w\":13,\"h\":9,\"i\":\"970e9ca1-f3a4-4660-8f3e-8879e53cc63e\"},\"panelIndex\":\"970e9ca1-f3a4-4660-8f3e-8879e53cc63e\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_2\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":18,\"w\":48,\"h\":15,\"i\":\"594e7e58-e32d-4a2d-b492-a97541a2a2d4\"},\"panelIndex\":\"594e7e58-e32d-4a2d-b492-a97541a2a2d4\",\"embeddableConfig\":{\"hidePanelTitles\":false,\"vis\":null},\"panelRefName\":\"panel_3\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":33,\"w\":48,\"h\":35,\"i\":\"aae9ec90-dea9-4a53-b4a7-99c5a6c91b55\"},\"panelIndex\":\"aae9ec90-dea9-4a53-b4a7-99c5a6c91b55\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"}]", "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":false}", "version": 1, @@ -61,8 +61,8 @@ "updated_at": "2022-06-13T13:57:55.163Z", "version": "Wzg5NiwyXQ==", "attributes": { - "title": "Malcolm Sensor Temperatures - Host Chooser", - "visState": "{\"title\":\"Malcolm Sensor Temperatures - Host Chooser\",\"type\":\"input_control_vis\",\"aggs\":[],\"params\":{\"controls\":[{\"id\":\"1655127648591\",\"fieldName\":\"host.name\",\"parent\":\"\",\"label\":\"Host Chooser\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"dynamicOptions\":true,\"size\":5,\"order\":\"desc\"},\"indexPatternRefName\":\"control_0_index_pattern\"}],\"updateFiltersOnChange\":false,\"useTimeFilter\":false,\"pinFilters\":false}}", + "title": "Hardware Temperature - Host Chooser", + "visState": "{\"title\":\"Hardware Temperature - Host Chooser\",\"type\":\"input_control_vis\",\"aggs\":[],\"params\":{\"controls\":[{\"id\":\"1655127648591\",\"fieldName\":\"host.name\",\"parent\":\"\",\"label\":\"Host Chooser\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"dynamicOptions\":true,\"size\":5,\"order\":\"desc\"},\"indexPatternRefName\":\"control_0_index_pattern\"}],\"updateFiltersOnChange\":false,\"useTimeFilter\":false,\"pinFilters\":false}}", "uiStateJSON": "{}", "description": "", "version": 1, @@ -90,8 +90,8 @@ "updated_at": "2022-06-13T14:26:58.711Z", "version": "WzkwOCwyXQ==", "attributes": { - "title": "Malcolm Sensor Temperature - Over Time", - "visState": "{\"title\":\"Malcolm Sensor Temperature - Over Time\",\"type\":\"line\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"params\":{\"field\":\"miscbeat.thermal.temp\",\"customLabel\":\"High °C\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-30m\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"host.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":8,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Host\"},\"schema\":\"group\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":true,\"valueAxis\":\"ValueAxis-1\"},\"labels\":{},\"legendPosition\":\"bottom\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"High °C\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"cardinal\",\"lineWidth\":1.5,\"mode\":\"normal\",\"show\":true,\"showCircles\":false,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"thresholdLine\":{\"color\":\"#E7664C\",\"show\":true,\"style\":\"full\",\"value\":95,\"width\":1},\"times\":[],\"type\":\"line\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":true,\"mode\":\"normal\",\"type\":\"linear\",\"setYExtents\":true,\"min\":35,\"max\":115},\"show\":true,\"style\":{},\"title\":{\"text\":\"High °C\"},\"type\":\"value\"}]}}", + "title": "Hardware Temperature - Over Time", + "visState": "{\"title\":\"Hardware Temperature - Over Time\",\"type\":\"line\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"params\":{\"field\":\"miscbeat.thermal.temp\",\"customLabel\":\"High °C\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-30m\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"host.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":8,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Host\"},\"schema\":\"group\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":true,\"valueAxis\":\"ValueAxis-1\"},\"labels\":{},\"legendPosition\":\"bottom\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"High °C\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"cardinal\",\"lineWidth\":1.5,\"mode\":\"normal\",\"show\":true,\"showCircles\":false,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"thresholdLine\":{\"color\":\"#E7664C\",\"show\":true,\"style\":\"full\",\"value\":95,\"width\":1},\"times\":[],\"type\":\"line\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":true,\"mode\":\"normal\",\"type\":\"linear\",\"setYExtents\":true,\"min\":35,\"max\":115},\"show\":true,\"style\":{},\"title\":{\"text\":\"High °C\"},\"type\":\"value\"}]}}", "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", "description": "", "version": 1, @@ -120,8 +120,8 @@ "updated_at": "2022-06-13T14:13:28.297Z", "version": "WzkwNCwyXQ==", "attributes": { - "title": "Malcolm Sensor Temprature - Sensor Chooser", - "visState": "{\"title\":\"Malcolm Sensor Temprature - Sensor Chooser\",\"type\":\"input_control_vis\",\"aggs\":[],\"params\":{\"controls\":[{\"id\":\"1655129580950\",\"fieldName\":\"miscbeat.thermal.type\",\"parent\":\"\",\"label\":\"Sensor Type\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"dynamicOptions\":true,\"size\":5,\"order\":\"desc\"},\"indexPatternRefName\":\"control_0_index_pattern\"}],\"updateFiltersOnChange\":false,\"useTimeFilter\":false,\"pinFilters\":false}}", + "title": "Hardware Temperature - Sensor Chooser", + "visState": "{\"title\":\"Hardware Temperature - Sensor Chooser\",\"type\":\"input_control_vis\",\"aggs\":[],\"params\":{\"controls\":[{\"id\":\"1655129580950\",\"fieldName\":\"miscbeat.thermal.type\",\"parent\":\"\",\"label\":\"Sensor Type\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"dynamicOptions\":true,\"size\":5,\"order\":\"desc\"},\"indexPatternRefName\":\"control_0_index_pattern\"}],\"updateFiltersOnChange\":false,\"useTimeFilter\":false,\"pinFilters\":false}}", "uiStateJSON": "{}", "description": "", "version": 1, @@ -149,8 +149,8 @@ "updated_at": "2022-06-13T14:24:35.386Z", "version": "WzkwNiwyXQ==", "attributes": { - "title": "Malcolm Sensor Temperature - Sensor Type", - "visState": "{\"title\":\"Malcolm Sensor Temperature - Sensor Type\",\"type\":\"gauge\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"params\":{\"field\":\"miscbeat.thermal.temp\",\"customLabel\":\" \"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.thermal.type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":6,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Sensor Type\"},\"schema\":\"group\"}],\"params\":{\"type\":\"gauge\",\"addTooltip\":true,\"addLegend\":false,\"isDisplayWarning\":false,\"gauge\":{\"alignment\":\"automatic\",\"extendRange\":true,\"percentageMode\":false,\"gaugeType\":\"Arc\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Labels\",\"colorsRange\":[{\"from\":0,\"to\":45},{\"from\":45,\"to\":60},{\"from\":60,\"to\":80},{\"from\":80,\"to\":90},{\"from\":90,\"to\":110}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"rgba(105,112,125,0.2)\"},\"type\":\"meter\",\"style\":{\"bgWidth\":0.9,\"width\":0.9,\"mask\":false,\"bgMask\":false,\"maskBars\":50,\"bgFill\":\"rgba(105,112,125,0.2)\",\"bgColor\":true,\"subText\":\"\",\"fontSize\":60}}}}", + "title": "Hardware Temperature - Sensor Type", + "visState": "{\"title\":\"Hardware Temperature - Sensor Type\",\"type\":\"gauge\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"params\":{\"field\":\"miscbeat.thermal.temp\",\"customLabel\":\" \"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.thermal.type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":6,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Sensor Type\"},\"schema\":\"group\"}],\"params\":{\"type\":\"gauge\",\"addTooltip\":true,\"addLegend\":false,\"isDisplayWarning\":false,\"gauge\":{\"alignment\":\"automatic\",\"extendRange\":true,\"percentageMode\":false,\"gaugeType\":\"Arc\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Labels\",\"colorsRange\":[{\"from\":0,\"to\":45},{\"from\":45,\"to\":60},{\"from\":60,\"to\":80},{\"from\":80,\"to\":90},{\"from\":90,\"to\":110}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"rgba(105,112,125,0.2)\"},\"type\":\"meter\",\"style\":{\"bgWidth\":0.9,\"width\":0.9,\"mask\":false,\"bgMask\":false,\"maskBars\":50,\"bgFill\":\"rgba(105,112,125,0.2)\",\"bgColor\":true,\"subText\":\"\",\"fontSize\":60}}}}", "uiStateJSON": "{}", "description": "", "version": 1, @@ -179,7 +179,7 @@ "updated_at": "2022-06-13T14:01:12.279Z", "version": "WzkwMSwyXQ==", "attributes": { - "title": "Malcolm Sensor Temperature - Logs", + "title": "Hardware Temperature - Logs", "description": "", "hits": 0, "columns": [ diff --git a/dashboards/dashboards/beats/Filebeat-nginx-logs.json b/dashboards/dashboards/beats/Filebeat-nginx-logs.json index 46f35dcdd..7e3bb7a02 100644 --- a/dashboards/dashboards/beats/Filebeat-nginx-logs.json +++ b/dashboards/dashboards/beats/Filebeat-nginx-logs.json @@ -10,8 +10,8 @@ "updated_at": "2022-06-01T19:53:27.884Z", "version": "WzEwMTgsMV0=", "attributes": { - "title": "Malcolm nginx Access and Error Logs", - "description": "Logs from Malcolm's own nginx instance", + "title": "nginx Access and Error Logs", + "description": "nginx Access and Error logs, including from Malcolm's own nginx instance", "hits": 0, "description": "", "panelsJSON": "[{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":16,\"w\":48,\"h\":15,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"columns\":[\"log.level\",\"error.message\"],\"sort\":[\"@timestamp\",\"desc\"]},\"panelRefName\":\"panel_0\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":31,\"w\":48,\"h\":23,\"i\":\"16\"},\"panelIndex\":\"16\",\"embeddableConfig\":{\"columns\":[\"url.original\",\"http.request.method\",\"http.response.status_code\",\"http.response.body.bytes\"],\"sort\":[\"@timestamp\",\"desc\"]},\"panelRefName\":\"panel_1\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":4,\"w\":48,\"h\":12,\"i\":\"18\"},\"panelIndex\":\"18\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":4,\"i\":\"19\"},\"panelIndex\":\"19\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"}]", diff --git a/dashboards/dashboards/beats/Filebeat-nginx-overview.json b/dashboards/dashboards/beats/Filebeat-nginx-overview.json index 4df0fb11c..ce8eed3fe 100644 --- a/dashboards/dashboards/beats/Filebeat-nginx-overview.json +++ b/dashboards/dashboards/beats/Filebeat-nginx-overview.json @@ -10,7 +10,7 @@ "updated_at": "2022-06-01T19:41:23.453Z", "version": "Wzg0NywxXQ==", "attributes": { - "description": "Overview of logs from Malcolm's own nginx instance", + "description": "nginx logs, including from Malcolm's own nginx instance", "hits": 0, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"version\":true,\"highlightAll\":true}" @@ -18,8 +18,7 @@ "optionsJSON": "{\"darkTheme\": false}", "panelsJSON": "[{\"panelIndex\":\"3\",\"panelRefName\":\"panel_0\",\"version\":\"7.3.0\",\"gridData\":{\"x\":36,\"y\":44,\"w\":12,\"h\":12,\"i\":\"3\"},\"embeddableConfig\":{}},{\"panelIndex\":\"4\",\"panelRefName\":\"panel_1\",\"version\":\"7.3.0\",\"gridData\":{\"x\":24,\"y\":44,\"w\":12,\"h\":12,\"i\":\"4\"},\"embeddableConfig\":{\"vis\":{\"legendOpen\":true}}},{\"panelIndex\":\"8\",\"panelRefName\":\"panel_2\",\"version\":\"7.3.0\",\"gridData\":{\"x\":0,\"y\":4,\"w\":48,\"h\":16,\"i\":\"8\"},\"embeddableConfig\":{\"mapZoom\":2,\"mapBounds\":{\"bottom_right\":{\"lat\":-7.362466865535738,\"lon\":245.39062500000003},\"top_left\":{\"lat\":77.07878389624943,\"lon\":-245.74218750000003}},\"mapCenter\":[50.51342652633956,-0.17578125],\"mapCollar\":{\"bottom_right\":{\"lat\":-49.583095,\"lon\":180},\"zoom\":2,\"top_left\":{\"lat\":90,\"lon\":-180}}}},{\"panelIndex\":\"13\",\"panelRefName\":\"panel_3\",\"version\":\"7.3.0\",\"gridData\":{\"x\":0,\"y\":20,\"w\":48,\"h\":12,\"i\":\"13\"},\"embeddableConfig\":{}},{\"panelIndex\":\"14\",\"panelRefName\":\"panel_4\",\"version\":\"7.3.0\",\"gridData\":{\"x\":24,\"y\":32,\"w\":24,\"h\":12,\"i\":\"14\"},\"embeddableConfig\":{}},{\"panelIndex\":\"15\",\"panelRefName\":\"panel_5\",\"version\":\"7.3.0\",\"gridData\":{\"x\":0,\"y\":32,\"w\":24,\"h\":12,\"i\":\"15\"},\"embeddableConfig\":{}},{\"panelIndex\":\"16\",\"panelRefName\":\"panel_6\",\"version\":\"7.3.0\",\"gridData\":{\"x\":0,\"y\":44,\"w\":24,\"h\":12,\"i\":\"16\"},\"embeddableConfig\":{}},{\"panelIndex\":\"17\",\"panelRefName\":\"panel_7\",\"version\":\"7.3.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":4,\"i\":\"17\"},\"embeddableConfig\":{}}]", "timeRestore": false, - "title": "Malcolm nginx Overview", - "description": "Overview of Malcolm's own nginx instance", + "title": "nginx Overview", "version": 1 }, "references": [ diff --git a/dashboards/dashboards/beats/Metricbeat-host-overview.json b/dashboards/dashboards/beats/Metricbeat-host-overview.json index fd140606b..a441acf0f 100644 --- a/dashboards/dashboards/beats/Metricbeat-host-overview.json +++ b/dashboards/dashboards/beats/Metricbeat-host-overview.json @@ -10,9 +10,9 @@ "updated_at": "2022-06-30T17:54:04.824Z", "version": "WzEwMzgsMV0=", "attributes": { - "title": "Malcolm Sensor Resources - Hosts Overview", + "title": "Resources - Hosts Overview", "hits": 0, - "description": "Malcolm sensor hosts' resources overview", + "description": "System resources hosts' level overview, including Malcolm sensors and aggregators", "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"1\",\"w\":24,\"x\":24,\"y\":57},\"panelIndex\":\"1\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"3\",\"w\":24,\"x\":0,\"y\":57},\"panelIndex\":\"3\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":17,\"i\":\"4\",\"w\":24,\"x\":24,\"y\":22},\"panelIndex\":\"4\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":17,\"i\":\"7\",\"w\":24,\"x\":24,\"y\":5},\"panelIndex\":\"7\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"8\",\"w\":24,\"x\":24,\"y\":39},\"panelIndex\":\"8\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":13,\"i\":\"10\",\"w\":12,\"x\":0,\"y\":5},\"panelIndex\":\"10\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_5\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":13,\"i\":\"11\",\"w\":12,\"x\":12,\"y\":5},\"panelIndex\":\"11\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_6\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":13,\"i\":\"12\",\"w\":12,\"x\":0,\"y\":30},\"panelIndex\":\"12\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_7\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":13,\"i\":\"13\",\"w\":12,\"x\":12,\"y\":30},\"panelIndex\":\"13\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_8\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":5,\"i\":\"16\",\"w\":24,\"x\":0,\"y\":0},\"panelIndex\":\"16\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_9\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":12,\"i\":\"21\",\"w\":8,\"x\":0,\"y\":18},\"panelIndex\":\"21\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_10\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":12,\"i\":\"22\",\"w\":8,\"x\":8,\"y\":18},\"panelIndex\":\"22\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_11\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":12,\"i\":\"23\",\"w\":8,\"x\":16,\"y\":18},\"panelIndex\":\"23\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_12\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"27\",\"w\":24,\"x\":0,\"y\":75},\"panelIndex\":\"27\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_13\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"28\",\"w\":24,\"x\":24,\"y\":75},\"panelIndex\":\"28\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_14\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":5,\"i\":\"30\",\"w\":24,\"x\":24,\"y\":0},\"panelIndex\":\"30\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_15\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":14,\"i\":\"b751999d-6577-4995-95e6-b7276ab6388d\",\"w\":24,\"x\":0,\"y\":43},\"panelIndex\":\"b751999d-6577-4995-95e6-b7276ab6388d\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_16\"}]", "optionsJSON": "{\"darkTheme\":false}", "version": 1, diff --git a/dashboards/dashboards/beats/Metricbeat-system-overview.json b/dashboards/dashboards/beats/Metricbeat-system-overview.json index e8bbf3ba9..407ccae86 100644 --- a/dashboards/dashboards/beats/Metricbeat-system-overview.json +++ b/dashboards/dashboards/beats/Metricbeat-system-overview.json @@ -10,9 +10,9 @@ "updated_at": "2022-06-30T17:45:03.314Z", "version": "Wzk2NCwxXQ==", "attributes": { - "title": "Malcolm Sensor Resources - System Overview", + "title": "Resources - System Overview", "hits": 0, - "description": "Malcolm sensor system resources overview", + "description": "System resources overview, including Malcolm sensors and aggregators", "panelsJSON": "[{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":5,\"i\":\"9\",\"w\":8,\"x\":0,\"y\":0},\"panelIndex\":\"9\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"gridData\":{\"h\":8,\"i\":\"11\",\"w\":8,\"x\":0,\"y\":5},\"panelIndex\":\"11\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"gridData\":{\"h\":20,\"i\":\"12\",\"w\":24,\"x\":24,\"y\":13},\"panelIndex\":\"12\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":20,\"i\":\"13\",\"w\":24,\"x\":0,\"y\":13},\"panelIndex\":\"13\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{\"vis\":{\"defaultColors\":{\"0% - 15%\":\"rgb(247,252,245)\",\"15% - 30%\":\"rgb(199,233,192)\",\"30% - 45%\":\"rgb(116,196,118)\",\"45% - 60%\":\"rgb(35,139,69)\"}}},\"gridData\":{\"h\":24,\"i\":\"14\",\"w\":48,\"x\":0,\"y\":33},\"panelIndex\":\"14\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"gridData\":{\"h\":13,\"i\":\"16\",\"w\":8,\"x\":32,\"y\":0},\"panelIndex\":\"16\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_5\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":13,\"i\":\"17\",\"w\":8,\"x\":40,\"y\":0},\"panelIndex\":\"17\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_6\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":13,\"i\":\"18\",\"w\":8,\"x\":24,\"y\":0},\"panelIndex\":\"18\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_7\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":13,\"i\":\"19\",\"w\":8,\"x\":16,\"y\":0},\"panelIndex\":\"19\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_8\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":13,\"i\":\"20\",\"w\":8,\"x\":8,\"y\":0},\"panelIndex\":\"20\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_9\"}]", "optionsJSON": "{\"darkTheme\":false}", "version": 1, diff --git a/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json b/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json index f465091b6..ca61e87d3 100644 --- a/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json +++ b/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json @@ -10,9 +10,9 @@ "updated_at": "2023-12-14T22:33:38.334Z", "version": "WzkzOCwxXQ==", "attributes": { - "title": "Malcolm Sensor Journald Logs", + "title": "Journald Logs", "hits": 0, - "description": "", + "description": "Messages from Systemd's Journald daemon on Linux environments, including Malcolm sensors and aggregators", "panelsJSON": "[{\"embeddableConfig\":{\"vis\":{\"legendOpen\":true}},\"gridData\":{\"h\":19,\"i\":\"b514b4e8-689b-465e-8335-ca20c20d46fe\",\"w\":14,\"x\":0,\"y\":0},\"panelIndex\":\"b514b4e8-689b-465e-8335-ca20c20d46fe\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":19,\"i\":\"7a7cfec2-8688-45a7-9790-66b3f0e9fd7e\",\"w\":11,\"x\":14,\"y\":0},\"panelIndex\":\"7a7cfec2-8688-45a7-9790-66b3f0e9fd7e\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":19,\"i\":\"2143906d-7adb-4de7-8484-2f87c8c98332\",\"w\":23,\"x\":25,\"y\":0},\"panelIndex\":\"2143906d-7adb-4de7-8484-2f87c8c98332\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"bb4d56fd-b110-4d58-b6aa-e4189bdba918\",\"w\":24,\"x\":0,\"y\":19},\"panelIndex\":\"bb4d56fd-b110-4d58-b6aa-e4189bdba918\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"4e4780cd-add6-4dbe-95ac-d11afbcd630c\",\"w\":24,\"x\":24,\"y\":19},\"panelIndex\":\"4e4780cd-add6-4dbe-95ac-d11afbcd630c\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":32,\"i\":\"77bbae8a-66cd-4e30-9b90-c7ab7c271320\",\"w\":48,\"x\":0,\"y\":37},\"panelIndex\":\"77bbae8a-66cd-4e30-9b90-c7ab7c271320\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_5\"}]", "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", "version": 1, @@ -66,8 +66,8 @@ "updated_at": "2023-12-14T22:33:38.334Z", "version": "WzkzOSwxXQ==", "attributes": { - "title": "Malcolm Sensor Journald - Logger", - "visState": "{\"title\":\"Malcolm Sensor Journald - Logger\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"log.logger\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100}}}", + "title": "Journald - Logger", + "visState": "{\"title\":\"Journald - Logger\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"log.logger\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100}}}", "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", "description": "", "version": 1, @@ -96,8 +96,8 @@ "updated_at": "2023-12-14T22:50:36.981Z", "version": "Wzk1NSwxXQ==", "attributes": { - "title": "Malcolm Sensor Journald - Process UID", - "visState": "{\"title\":\"Malcolm Sensor Journald - Process UID\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.hostname\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd Host\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"process.user.id\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Process UID\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "title": "Journald - Process UID", + "visState": "{\"title\":\"Journald - Process UID\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.hostname\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd Host\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"process.user.id\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Process UID\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", "uiStateJSON": "{\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}}}", "description": "", "version": 1, @@ -126,8 +126,8 @@ "updated_at": "2023-12-14T22:33:38.334Z", "version": "Wzk0MSwxXQ==", "attributes": { - "title": "Malcolm Sensor Journald - Logs by Host Over Time", - "visState": "{\"title\":\"Malcolm Sensor Journald - Logs by Host Over Time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.hostname\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Journald Host\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":true,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", + "title": "Journald - Logs by Host Over Time", + "visState": "{\"title\":\"Journald - Logs by Host Over Time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.hostname\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Journald Host\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":true,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", "uiStateJSON": "{}", "description": "", "version": 1, @@ -156,8 +156,8 @@ "updated_at": "2023-12-14T22:33:38.334Z", "version": "Wzk0MiwxXQ==", "attributes": { - "title": "Malcolm Sensor Journald - Systemd Unit", - "visState": "{\"title\":\"Malcolm Sensor Journald - Systemd Unit\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.systemd_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd Unit\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.systemd_user_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd User Unit\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.user_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"User Unit\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "title": "Journald - Systemd Unit", + "visState": "{\"title\":\"Journald - Systemd Unit\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.systemd_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd Unit\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.systemd_user_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd User Unit\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.user_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"User Unit\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", "uiStateJSON": "{\"vis\":{\"sortColumn\":{\"colIndex\":3,\"direction\":\"desc\"}}}", "description": "", "version": 1, @@ -186,8 +186,8 @@ "updated_at": "2023-12-14T22:33:38.334Z", "version": "Wzk0MywxXQ==", "attributes": { - "title": "Malcolm Sensor Journald - Process Name", - "visState": "{\"title\":\"Malcolm Sensor Journald - Process Name\",\"type\":\"tagcloud\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"process.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"scale\":\"square root\",\"orientation\":\"single\",\"minFontSize\":18,\"maxFontSize\":72,\"showLabel\":false}}", + "title": "Journald - Process Name", + "visState": "{\"title\":\"Journald - Process Name\",\"type\":\"tagcloud\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"process.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"scale\":\"square root\",\"orientation\":\"single\",\"minFontSize\":18,\"maxFontSize\":72,\"showLabel\":false}}", "uiStateJSON": "{}", "description": "", "version": 1, @@ -216,7 +216,7 @@ "updated_at": "2023-12-14T22:33:38.334Z", "version": "Wzk0NCwxXQ==", "attributes": { - "title": "Malcolm Sensor Journald - Logs", + "title": "Journald - Logs", "description": "", "hits": 0, "columns": [ diff --git a/dashboards/opensearch_dashboards.yml b/dashboards/opensearch_dashboards.yml index 2a7f64fe3..e8159076b 100644 --- a/dashboards/opensearch_dashboards.yml +++ b/dashboards/opensearch_dashboards.yml @@ -26,7 +26,7 @@ opensearchDashboards.branding: loadingLogo: defaultUrl: "/assets/img/icon.png" darkModeUrl: "/assets/img/icon_dark.png" - faviconUrl: "/assets/favicon.ico" + faviconUrl: "/assets/img/favicon32.png" map.regionmap: includeOpenSearchMapsService: false diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index 034c7e569..7f34c7f74 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -1526,7 +1526,19 @@ filter { # intel.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Info - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_intel_fields" + rename => { "[zeek_cols][cif.firstseen]" => "[zeek_cols][cif_firstseen]" } + rename => { "[zeek_cols][cif.lastseen]" => "[zeek_cols][cif_lastseen]" } + rename => { "[zeek_cols][cif.tags]" => "[zeek_cols][cif_tags]" } + rename => { "[zeek_cols][seen.indicator]" => "[zeek_cols][seen_indicator]" } + rename => { "[zeek_cols][seen.indicator_type]" => "[zeek_cols][seen_indicator_type]" } + rename => { "[zeek_cols][seen.node]" => "[zeek_cols][seen_node]" } + rename => { "[zeek_cols][seen.where]" => "[zeek_cols][seen_where]" } + } + + } else { dissect { id => "dissect_zeek_intel" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -1548,10 +1560,15 @@ filter { } mutate { id => "mutate_split_zeek_intel_commas" split => { "[zeek_cols][sources]" => "," - "[zeek_cols][matched]" => "," - "[zeek_cols][cif_tags]" => "," } } + "[zeek_cols][matched]" => "," } } } + # For some reason, even in JSON, I have cif_tags strings like: + # Network activity,osint:source-type=\"block-or-filter-list\" + # so whatever reason it's not already an array. Split it here. + mutate { id => "mutate_split_zeek_intel_cif_tags" + split => { "[zeek_cols][cif_tags]" => "," } } + } else if ([log_source] == "ipsec") { ############################################################################################################################# # ipsec.log From 5b4e0706fcb93017e7819ac94e16fccbb15526a1 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 29 Feb 2024 14:43:29 -0700 Subject: [PATCH 64/79] minor tweaks as preparing for v24.03.0 --- .../composable/component/miscbeat.json | 27 +++++++++++++++++++ logstash/pipelines/beats/11_beats_logs.conf | 23 ++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/dashboards/templates/composable/component/miscbeat.json b/dashboards/templates/composable/component/miscbeat.json index d3072b3c4..0e309d6a5 100644 --- a/dashboards/templates/composable/component/miscbeat.json +++ b/dashboards/templates/composable/component/miscbeat.json @@ -93,6 +93,33 @@ "properties": { } }, + "kmsg": { + "properties": { + "priority": { "type": "integer" }, + "sequence": { "type": "long" }, + "sec": { "type": "long" }, + "usec": { "type": "integer" } + } + }, + "proc": { + "properties": { + "alive": { "type": "keyword" }, + "proc_name": { "type": "keyword" }, + "pid": { "type": "long" }, + "mem.VmPeak": { "type": "long" }, + "mem.VmSize": { "type": "long" }, + "mem.VmLck": { "type": "long" }, + "mem.VmHWM": { "type": "long" }, + "mem.VmRSS": { "type": "long" }, + "mem.VmData": { "type": "long" }, + "mem.VmStk": { "type": "long" }, + "mem.VmExe": { "type": "long" }, + "mem.VmLib": { "type": "long" }, + "mem.VmPTE": { "type": "long" }, + "mem.VmSwap": { "type": "long" }, + "fd": { "type": "integer" } + } + }, "systemd": { "properties": { "audit_loginuid": { "type": "integer" }, diff --git a/logstash/pipelines/beats/11_beats_logs.conf b/logstash/pipelines/beats/11_beats_logs.conf index 7f883c163..979529161 100644 --- a/logstash/pipelines/beats/11_beats_logs.conf +++ b/logstash/pipelines/beats/11_beats_logs.conf @@ -492,6 +492,29 @@ filter { } # if ([miscbeat][network]) + if ([miscbeat][proc]) { + #------------------------------------------------- + # proc - https://docs.fluentbit.io/manual/pipeline/inputs/process + + if (![event][hash]) { + fingerprint { + id => "fingerprint_malcolm_miscbeat_proc" + source => [ "[host][name]", + "[event][module]", + "[miscbeat][proc][proc_name]", + "[miscbeat][proc][pid]", + "[miscbeat][proc][fd]", + "[@timestamp]" ] + concatenate_sources => true + # uses event.hash + ecs_compatibility => "v8" + method => "MURMUR3_128" + base64encode => true + } + } + + } # if ([miscbeat][proc]) + if ([miscbeat][syslog]) { #------------------------------------------------- # syslog - https://docs.fluentbit.io/manual/pipeline/inputs/syslog From 34dd3bd25b364b60743dcd78490a28b65831d49c Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 1 Mar 2024 09:14:33 -0700 Subject: [PATCH 65/79] Work in progress for idaholab/Malcolm#414, capture stats dashboard for zeek --- .../4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json | 459 ++++++++++++++++++ 1 file changed, 459 insertions(+) create mode 100644 dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json diff --git a/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json b/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json new file mode 100644 index 000000000..8106e0924 --- /dev/null +++ b/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json @@ -0,0 +1,459 @@ +{ + "version": "2.12.0", + "objects": [ + { + "id": "4ca94c70-d7da-11ee-9ed3-e7afff29e59a", + "type": "dashboard", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T16:14:00.110Z", + "version": "Wzk3NiwxXQ==", + "attributes": { + "title": "Packet Capture - Zeek", + "hits": 0, + "description": "", + "panelsJSON": "[{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":16,\"i\":\"b483d809-a528-4280-b79e-aa7ada17d275\"},\"panelIndex\":\"b483d809-a528-4280-b79e-aa7ada17d275\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":16,\"w\":13,\"h\":9,\"i\":\"3d3ec563-b930-44bb-9e00-854f019b2263\"},\"panelIndex\":\"3d3ec563-b930-44bb-9e00-854f019b2263\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_1\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":16,\"w\":14,\"h\":10,\"i\":\"e10dc0a6-f197-4cbc-a1ad-e67194f95a63\"},\"panelIndex\":\"e10dc0a6-f197-4cbc-a1ad-e67194f95a63\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_2\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":27,\"y\":16,\"w\":21,\"h\":27,\"i\":\"8e013ce7-3205-4d06-a805-6285826c1c5d\"},\"panelIndex\":\"8e013ce7-3205-4d06-a805-6285826c1c5d\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":25,\"w\":13,\"h\":18,\"i\":\"74a841b8-2ffc-4f6d-8b5a-ca7960eb6b10\"},\"panelIndex\":\"74a841b8-2ffc-4f6d-8b5a-ca7960eb6b10\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":26,\"w\":14,\"h\":10,\"i\":\"01b20859-4d95-47e0-a536-6b1e9932c35b\"},\"panelIndex\":\"01b20859-4d95-47e0-a536-6b1e9932c35b\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_5\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":36,\"w\":14,\"h\":7,\"i\":\"83b89475-80bb-4e6c-a9dd-c7f41a74c88c\"},\"panelIndex\":\"83b89475-80bb-4e6c-a9dd-c7f41a74c88c\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_6\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":43,\"w\":48,\"h\":21,\"i\":\"36e03a4a-e017-42b8-82cf-205d26b2ed6b\"},\"panelIndex\":\"36e03a4a-e017-42b8-82cf-205d26b2ed6b\",\"embeddableConfig\":{},\"panelRefName\":\"panel_7\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":64,\"w\":48,\"h\":20,\"i\":\"f15e46fe-040f-4602-ad13-01aab36b372a\"},\"panelIndex\":\"f15e46fe-040f-4602-ad13-01aab36b372a\",\"embeddableConfig\":{},\"panelRefName\":\"panel_8\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":84,\"w\":16,\"h\":17,\"i\":\"bfdc6d50-66f1-4f9a-9ea5-cd30bc01099d\"},\"panelIndex\":\"bfdc6d50-66f1-4f9a-9ea5-cd30bc01099d\",\"embeddableConfig\":{},\"panelRefName\":\"panel_9\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":16,\"y\":84,\"w\":32,\"h\":17,\"i\":\"efbd7f15-5af7-4e39-9889-c1c944a40dc2\"},\"panelIndex\":\"efbd7f15-5af7-4e39-9889-c1c944a40dc2\",\"embeddableConfig\":{\"columns\":[\"@timestamp\",\"host.name\",\"zeek.reporter.level\",\"zeek.reporter.message\",\"zeek.reporter.location\"]},\"panelRefName\":\"panel_10\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":101,\"w\":48,\"h\":17,\"i\":\"cd0f831b-205a-4791-9a7b-0b4e7440c4fd\"},\"panelIndex\":\"cd0f831b-205a-4791-9a7b-0b4e7440c4fd\",\"embeddableConfig\":{},\"panelRefName\":\"panel_11\"}]", + "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", + "version": 1, + "timeRestore": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}" + } + }, + "references": [ + { + "name": "panel_0", + "type": "visualization", + "id": "611a3e20-d7e0-11ee-b25e-e793ed358448" + }, + { + "name": "panel_1", + "type": "visualization", + "id": "ba90d2c0-d7e5-11ee-b25e-e793ed358448" + }, + { + "name": "panel_2", + "type": "visualization", + "id": "13facbf0-d7e5-11ee-b25e-e793ed358448" + }, + { + "name": "panel_3", + "type": "search", + "id": "991158b0-d7d9-11ee-9ed3-e7afff29e59a" + }, + { + "name": "panel_4", + "type": "visualization", + "id": "abcfca50-d7dd-11ee-b25e-e793ed358448" + }, + { + "name": "panel_5", + "type": "visualization", + "id": "8051d2c0-d7e6-11ee-b25e-e793ed358448" + }, + { + "name": "panel_6", + "type": "visualization", + "id": "65b7efe0-d7e5-11ee-b25e-e793ed358448" + }, + { + "name": "panel_7", + "type": "search", + "id": "9c6a1610-d7d8-11ee-9ed3-e7afff29e59a" + }, + { + "name": "panel_8", + "type": "search", + "id": "351ef380-d7d9-11ee-9ed3-e7afff29e59a" + }, + { + "name": "panel_9", + "type": "visualization", + "id": "c75b6df0-d7e3-11ee-b25e-e793ed358448" + }, + { + "name": "panel_10", + "type": "search", + "id": "17619ea0-d7da-11ee-9ed3-e7afff29e59a" + }, + { + "name": "panel_11", + "type": "search", + "id": "e7047c50-d7d9-11ee-9ed3-e7afff29e59a" + } + ], + "migrationVersion": { + "dashboard": "7.9.3" + } + }, + { + "id": "611a3e20-d7e0-11ee-b25e-e793ed358448", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T15:49:26.992Z", + "version": "Wzk2NywxXQ==", + "attributes": { + "title": "Zeek Capture Measurements ", + "visState": "{\"title\":\"Zeek Capture Measurements \",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"32d1fca0-d7e1-11ee-ad81-217e54128a4b\",\"color\":\"rgba(17,255,0,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"32d1fca1-d7e1-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_link\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Packets seen\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"32d1fca1-d7e1-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"rgba(84,179,153,1)\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"zeek.capture_loss.acks\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"ACKs seen\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_order_by\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"terms_size\":\"25\"},{\"id\":\"e4143600-d7e0-11ee-ad81-217e54128a4b\",\"color\":\"rgba(175,8,8,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_dropped\"},{\"id\":\"f6df2790-d7e0-11ee-ad81-217e54128a4b\",\"type\":\"math\",\"variables\":[{\"id\":\"f8ee0a60-d7e0-11ee-ad81-217e54128a4b\",\"name\":\"packets\",\"field\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\"}],\"script\":\"params.packets*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Packets dropped\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"20b9a420-d7df-11ee-ad81-217e54128a4b\",\"color\":\"rgba(211,96,134,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.capture_loss.gaps\"},{\"id\":\"9a3afce0-d7df-11ee-ad81-217e54128a4b\",\"type\":\"math\",\"variables\":[{\"id\":\"9dece150-d7df-11ee-ad81-217e54128a4b\",\"name\":\"gaps\",\"field\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\"}],\"script\":\"params.gaps*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"ACKS missed\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"filter\":{\"query\":\"event.provider:zeek\",\"language\":\"kuery\"},\"legend_position\":\"right\",\"background_color\":null}}", + "uiStateJSON": "{}", + "description": "Positive values on the y-axis represent observed traffic (packets and ACKs), while negative values represent missing dropped packets and missing ACKs.\n\nThis data is logged by Zeek in stats.log (https://docs.zeek.org/en/master/scripts/policy/misc/stats.zeek.html#type-Stats::Info) and capture_loss.log (https://docs.zeek.org/en/master/scripts/policy/misc/capture-loss.zeek.html#type-CaptureLoss::Info).", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "ba90d2c0-d7e5-11ee-b25e-e793ed358448", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T16:06:57.260Z", + "version": "Wzk3MSwxXQ==", + "attributes": { + "title": "Zeek Capture - Host Filter", + "visState": "{\"title\":\"Zeek Capture - Host Filter\",\"type\":\"input_control_vis\",\"aggs\":[],\"params\":{\"controls\":[{\"id\":\"1709309167905\",\"fieldName\":\"host.name\",\"parent\":\"\",\"label\":\"Capture Host\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"dynamicOptions\":true,\"size\":5,\"order\":\"desc\"},\"indexPatternRefName\":\"control_0_index_pattern\"}],\"updateFiltersOnChange\":false,\"useTimeFilter\":false,\"pinFilters\":true}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + }, + "references": [ + { + "name": "control_0_index_pattern", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "13facbf0-d7e5-11ee-b25e-e793ed358448", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T16:02:17.775Z", + "version": "Wzk2OSwxXQ==", + "attributes": { + "title": "Zeek Stats - Packets and Bytes", + "visState": "{\"title\":\"Zeek Stats - Packets and Bytes\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_link\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Total Packets Seen\"},{\"id\":\"bd4560e0-d7e4-11ee-ad81-217e54128a4b\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"bd4560e1-d7e4-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.bytes_recv\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Total Bytes Seen\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:zeek AND event.dataset:stats\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "991158b0-d7d9-11ee-9ed3-e7afff29e59a", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T15:01:44.258Z", + "version": "Wzg4MywxXQ==", + "attributes": { + "title": "Packet Capture - Zeek capture_loss.log", + "description": "", + "hits": 0, + "columns": [ + "zeek.capture_loss.ts_delta", + "host.name", + "zeek.capture_loss.peer", + "zeek.capture_loss.acks", + "zeek.capture_loss.gaps", + "zeek.capture_loss.percent_lost" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.provider:zeek and event.dataset:capture_loss\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + }, + { + "id": "abcfca50-d7dd-11ee-b25e-e793ed358448", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T15:09:16.533Z", + "version": "Wzk1OSwxXQ==", + "attributes": { + "title": "Zeek Analyzer Messages", + "visState": "{\"title\":\"Zeek Analyzer Messages\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.cause\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Cause\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.analyzer_kind\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Class\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.analyzer_name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Analyzer\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "351ef380-d7d9-11ee-9ed3-e7afff29e59a" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "8051d2c0-d7e6-11ee-b25e-e793ed358448", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T16:12:29.036Z", + "version": "Wzk3NCwxXQ==", + "attributes": { + "title": "Zeek Stats - Capture Loss", + "visState": "{\"title\":\"Zeek Stats - Capture Loss\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_dropped\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Packets Dropped\"},{\"id\":\"bd4560e0-d7e4-11ee-ad81-217e54128a4b\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"bd4560e1-d7e4-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.capture_loss.gaps\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"ACKs Missed\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:zeek AND event.dataset:(stats OR capture_loss)\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "65b7efe0-d7e5-11ee-b25e-e793ed358448", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T16:04:34.910Z", + "version": "Wzk3MCwxXQ==", + "attributes": { + "title": "Zeek Stats - Files", + "visState": "{\"title\":\"Zeek Stats - Files\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"zeek.stats.files\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Files Seen\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"400f2100-d7e5-11ee-ad81-217e54128a4b\"}],\"time_range_mode\":\"entire_time_range\",\"filter\":{\"query\":\"event.provider:zeek AND event.dataset:stats\",\"language\":\"kuery\"}}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "9c6a1610-d7d8-11ee-9ed3-e7afff29e59a", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T15:01:44.258Z", + "version": "Wzg4NCwxXQ==", + "attributes": { + "title": "Packet Capture - Zeek stats.log", + "description": "", + "hits": 0, + "columns": [ + "host.name", + "zeek.stats.peer", + "zeek.stats.mem", + "zeek.stats.pkts_link", + "zeek.stats.pkts_proc", + "zeek.stats.pkts_dropped", + "zeek.stats.bytes_recv", + "zeek.stats.tcp_conns", + "zeek.stats.udp_conns", + "zeek.stats.icmp_conns", + "zeek.stats.files" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.provider:zeek and event.dataset:stats\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + }, + { + "id": "351ef380-d7d9-11ee-9ed3-e7afff29e59a", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T15:01:44.258Z", + "version": "Wzg4MiwxXQ==", + "attributes": { + "title": "Packet Capture - Zeek analyzer.log", + "description": "", + "hits": 0, + "columns": [ + "host.name", + "zeek.analyzer.analyzer_kind", + "zeek.analyzer.analyzer_name", + "zeek.analyzer.cause", + "zeek.analyzer.failure_reason", + "source.ip", + "destination.ip", + "event.id" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.provider:zeek and event.dataset:analyzer\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"firstPacket\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "arkime_sessions3-*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + }, + { + "id": "c75b6df0-d7e3-11ee-b25e-e793ed358448", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T15:52:59.726Z", + "version": "Wzk2OCwxXQ==", + "attributes": { + "title": "Zeek - Reporter Categories", + "visState": "{\"title\":\"Zeek - Reporter Categories\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.reporter.level.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100}}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "17619ea0-d7da-11ee-9ed3-e7afff29e59a" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "17619ea0-d7da-11ee-9ed3-e7afff29e59a", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T15:01:44.258Z", + "version": "Wzg4NSwxXQ==", + "attributes": { + "title": "Packet Capture - Zeek reporter.log", + "description": "", + "hits": 0, + "columns": [ + "host.name", + "zeek.reporter.level", + "zeek.reporter.location", + "zeek.reporter.message" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.provider:zeek and event.dataset:reporter\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30d\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + }, + { + "id": "e7047c50-d7d9-11ee-9ed3-e7afff29e59a", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T15:01:44.258Z", + "version": "Wzg4NiwxXQ==", + "attributes": { + "title": "Packet Capture - Zeek cluster.log", + "description": "", + "hits": 0, + "columns": [ + "host.name", + "zeek.cluster.node", + "zeek.cluster.message" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.provider:zeek and event.dataset:cluster\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30d\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + } + ] +} \ No newline at end of file From 5aa8e9d403422efd3f55f27fee7c28cd2f472fee Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 1 Mar 2024 12:20:05 -0700 Subject: [PATCH 66/79] Work in progress for idaholab/Malcolm#414, capture stats dashboard for zeek --- .../4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json | 265 ++++++++++++------ 1 file changed, 178 insertions(+), 87 deletions(-) diff --git a/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json b/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json index 8106e0924..1ed2da6aa 100644 --- a/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json +++ b/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json @@ -7,13 +7,13 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T16:14:00.110Z", - "version": "Wzk3NiwxXQ==", + "updated_at": "2024-03-01T19:19:33.901Z", + "version": "WzEwOTksMV0=", "attributes": { - "title": "Packet Capture - Zeek", + "title": "Packet Capture Statistics", "hits": 0, - "description": "", - "panelsJSON": "[{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":16,\"i\":\"b483d809-a528-4280-b79e-aa7ada17d275\"},\"panelIndex\":\"b483d809-a528-4280-b79e-aa7ada17d275\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":16,\"w\":13,\"h\":9,\"i\":\"3d3ec563-b930-44bb-9e00-854f019b2263\"},\"panelIndex\":\"3d3ec563-b930-44bb-9e00-854f019b2263\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_1\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":16,\"w\":14,\"h\":10,\"i\":\"e10dc0a6-f197-4cbc-a1ad-e67194f95a63\"},\"panelIndex\":\"e10dc0a6-f197-4cbc-a1ad-e67194f95a63\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_2\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":27,\"y\":16,\"w\":21,\"h\":27,\"i\":\"8e013ce7-3205-4d06-a805-6285826c1c5d\"},\"panelIndex\":\"8e013ce7-3205-4d06-a805-6285826c1c5d\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":25,\"w\":13,\"h\":18,\"i\":\"74a841b8-2ffc-4f6d-8b5a-ca7960eb6b10\"},\"panelIndex\":\"74a841b8-2ffc-4f6d-8b5a-ca7960eb6b10\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":26,\"w\":14,\"h\":10,\"i\":\"01b20859-4d95-47e0-a536-6b1e9932c35b\"},\"panelIndex\":\"01b20859-4d95-47e0-a536-6b1e9932c35b\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_5\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":36,\"w\":14,\"h\":7,\"i\":\"83b89475-80bb-4e6c-a9dd-c7f41a74c88c\"},\"panelIndex\":\"83b89475-80bb-4e6c-a9dd-c7f41a74c88c\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_6\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":43,\"w\":48,\"h\":21,\"i\":\"36e03a4a-e017-42b8-82cf-205d26b2ed6b\"},\"panelIndex\":\"36e03a4a-e017-42b8-82cf-205d26b2ed6b\",\"embeddableConfig\":{},\"panelRefName\":\"panel_7\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":64,\"w\":48,\"h\":20,\"i\":\"f15e46fe-040f-4602-ad13-01aab36b372a\"},\"panelIndex\":\"f15e46fe-040f-4602-ad13-01aab36b372a\",\"embeddableConfig\":{},\"panelRefName\":\"panel_8\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":84,\"w\":16,\"h\":17,\"i\":\"bfdc6d50-66f1-4f9a-9ea5-cd30bc01099d\"},\"panelIndex\":\"bfdc6d50-66f1-4f9a-9ea5-cd30bc01099d\",\"embeddableConfig\":{},\"panelRefName\":\"panel_9\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":16,\"y\":84,\"w\":32,\"h\":17,\"i\":\"efbd7f15-5af7-4e39-9889-c1c944a40dc2\"},\"panelIndex\":\"efbd7f15-5af7-4e39-9889-c1c944a40dc2\",\"embeddableConfig\":{\"columns\":[\"@timestamp\",\"host.name\",\"zeek.reporter.level\",\"zeek.reporter.message\",\"zeek.reporter.location\"]},\"panelRefName\":\"panel_10\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":101,\"w\":48,\"h\":17,\"i\":\"cd0f831b-205a-4791-9a7b-0b4e7440c4fd\"},\"panelIndex\":\"cd0f831b-205a-4791-9a7b-0b4e7440c4fd\",\"embeddableConfig\":{},\"panelRefName\":\"panel_11\"}]", + "description": "Statistics and diagnostics for packet capture from Zeek and Suricata", + "panelsJSON": "[{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":13,\"h\":15,\"i\":\"0c179e97-9bcf-4f72-b717-b7a93667c1a0\"},\"panelIndex\":\"0c179e97-9bcf-4f72-b717-b7a93667c1a0\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":0,\"w\":35,\"h\":15,\"i\":\"b483d809-a528-4280-b79e-aa7ada17d275\"},\"panelIndex\":\"b483d809-a528-4280-b79e-aa7ada17d275\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_1\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":15,\"w\":13,\"h\":10,\"i\":\"e10dc0a6-f197-4cbc-a1ad-e67194f95a63\"},\"panelIndex\":\"e10dc0a6-f197-4cbc-a1ad-e67194f95a63\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_2\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":15,\"w\":13,\"h\":10,\"i\":\"01b20859-4d95-47e0-a536-6b1e9932c35b\"},\"panelIndex\":\"01b20859-4d95-47e0-a536-6b1e9932c35b\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_3\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":26,\"y\":15,\"w\":22,\"h\":20,\"i\":\"8e013ce7-3205-4d06-a805-6285826c1c5d\"},\"panelIndex\":\"8e013ce7-3205-4d06-a805-6285826c1c5d\",\"embeddableConfig\":{\"columns\":[\"@timestamp\",\"host.name\",\"zeek.capture_loss.peer\",\"zeek.capture_loss.acks\",\"zeek.capture_loss.gaps\",\"zeek.capture_loss.percent_lost\"]},\"panelRefName\":\"panel_4\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":25,\"w\":13,\"h\":10,\"i\":\"147b45ae-804b-4d9e-a9a9-806772ad3b35\"},\"panelIndex\":\"147b45ae-804b-4d9e-a9a9-806772ad3b35\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_5\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":25,\"w\":13,\"h\":10,\"i\":\"687597e3-4848-4629-8b85-45c0773efb79\"},\"panelIndex\":\"687597e3-4848-4629-8b85-45c0773efb79\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_6\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":35,\"w\":24,\"h\":15,\"i\":\"0174654c-2010-463a-b49e-fa5759b61b9c\"},\"panelIndex\":\"0174654c-2010-463a-b49e-fa5759b61b9c\",\"embeddableConfig\":{},\"panelRefName\":\"panel_7\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":50,\"w\":48,\"h\":21,\"i\":\"36e03a4a-e017-42b8-82cf-205d26b2ed6b\"},\"panelIndex\":\"36e03a4a-e017-42b8-82cf-205d26b2ed6b\",\"embeddableConfig\":{},\"panelRefName\":\"panel_8\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":71,\"w\":48,\"h\":21,\"i\":\"e1c0f1e0-de36-4527-bafa-a297fe9452a2\"},\"panelIndex\":\"e1c0f1e0-de36-4527-bafa-a297fe9452a2\",\"embeddableConfig\":{},\"panelRefName\":\"panel_9\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":92,\"w\":13,\"h\":20,\"i\":\"74a841b8-2ffc-4f6d-8b5a-ca7960eb6b10\"},\"panelIndex\":\"74a841b8-2ffc-4f6d-8b5a-ca7960eb6b10\",\"embeddableConfig\":{},\"panelRefName\":\"panel_10\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":92,\"w\":35,\"h\":20,\"i\":\"f15e46fe-040f-4602-ad13-01aab36b372a\"},\"panelIndex\":\"f15e46fe-040f-4602-ad13-01aab36b372a\",\"embeddableConfig\":{},\"panelRefName\":\"panel_11\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":112,\"w\":16,\"h\":17,\"i\":\"bfdc6d50-66f1-4f9a-9ea5-cd30bc01099d\"},\"panelIndex\":\"bfdc6d50-66f1-4f9a-9ea5-cd30bc01099d\",\"embeddableConfig\":{},\"panelRefName\":\"panel_12\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":16,\"y\":112,\"w\":32,\"h\":17,\"i\":\"efbd7f15-5af7-4e39-9889-c1c944a40dc2\"},\"panelIndex\":\"efbd7f15-5af7-4e39-9889-c1c944a40dc2\",\"embeddableConfig\":{\"columns\":[\"@timestamp\",\"host.name\",\"zeek.reporter.level\",\"zeek.reporter.message\",\"zeek.reporter.location\"]},\"panelRefName\":\"panel_13\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":24,\"y\":35,\"w\":24,\"h\":15,\"i\":\"2ecc4ac3-d694-46ab-a6b1-9c86e5e9d394\"},\"panelIndex\":\"2ecc4ac3-d694-46ab-a6b1-9c86e5e9d394\",\"embeddableConfig\":{},\"panelRefName\":\"panel_14\"}]", "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", "version": 1, "timeRestore": false, @@ -25,12 +25,12 @@ { "name": "panel_0", "type": "visualization", - "id": "611a3e20-d7e0-11ee-b25e-e793ed358448" + "id": "a1480cb0-d7ff-11ee-b25e-e793ed358448" }, { "name": "panel_1", "type": "visualization", - "id": "ba90d2c0-d7e5-11ee-b25e-e793ed358448" + "id": "611a3e20-d7e0-11ee-b25e-e793ed358448" }, { "name": "panel_2", @@ -39,48 +39,63 @@ }, { "name": "panel_3", - "type": "search", - "id": "991158b0-d7d9-11ee-9ed3-e7afff29e59a" + "type": "visualization", + "id": "8051d2c0-d7e6-11ee-b25e-e793ed358448" }, { "name": "panel_4", - "type": "visualization", - "id": "abcfca50-d7dd-11ee-b25e-e793ed358448" + "type": "search", + "id": "991158b0-d7d9-11ee-9ed3-e7afff29e59a" }, { "name": "panel_5", "type": "visualization", - "id": "8051d2c0-d7e6-11ee-b25e-e793ed358448" + "id": "cc976b70-d7fc-11ee-b25e-e793ed358448" }, { "name": "panel_6", "type": "visualization", - "id": "65b7efe0-d7e5-11ee-b25e-e793ed358448" + "id": "9833e6a0-d7fd-11ee-b25e-e793ed358448" }, { "name": "panel_7", + "type": "visualization", + "id": "6b7b9a40-faa1-11e6-86b1-cd7735ff7e23-ecs" + }, + { + "name": "panel_8", "type": "search", "id": "9c6a1610-d7d8-11ee-9ed3-e7afff29e59a" }, { - "name": "panel_8", + "name": "panel_9", + "type": "search", + "id": "b0aabf40-d7fa-11ee-b25e-e793ed358448" + }, + { + "name": "panel_10", + "type": "visualization", + "id": "abcfca50-d7dd-11ee-b25e-e793ed358448" + }, + { + "name": "panel_11", "type": "search", "id": "351ef380-d7d9-11ee-9ed3-e7afff29e59a" }, { - "name": "panel_9", + "name": "panel_12", "type": "visualization", "id": "c75b6df0-d7e3-11ee-b25e-e793ed358448" }, { - "name": "panel_10", + "name": "panel_13", "type": "search", "id": "17619ea0-d7da-11ee-9ed3-e7afff29e59a" }, { - "name": "panel_11", - "type": "search", - "id": "e7047c50-d7d9-11ee-9ed3-e7afff29e59a" + "name": "panel_14", + "type": "visualization", + "id": "089b85d0-1b16-11e7-b09e-037021c4f8df-ecs" } ], "migrationVersion": { @@ -88,53 +103,53 @@ } }, { - "id": "611a3e20-d7e0-11ee-b25e-e793ed358448", + "id": "a1480cb0-d7ff-11ee-b25e-e793ed358448", "type": "visualization", "namespaces": [ "default" ], - "updated_at": "2024-03-01T15:49:26.992Z", - "version": "Wzk2NywxXQ==", + "updated_at": "2024-03-01T19:12:21.755Z", + "version": "WzEwOTUsMV0=", "attributes": { - "title": "Zeek Capture Measurements ", - "visState": "{\"title\":\"Zeek Capture Measurements \",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"32d1fca0-d7e1-11ee-ad81-217e54128a4b\",\"color\":\"rgba(17,255,0,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"32d1fca1-d7e1-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_link\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Packets seen\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"32d1fca1-d7e1-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"rgba(84,179,153,1)\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"zeek.capture_loss.acks\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"ACKs seen\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_order_by\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"terms_size\":\"25\"},{\"id\":\"e4143600-d7e0-11ee-ad81-217e54128a4b\",\"color\":\"rgba(175,8,8,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_dropped\"},{\"id\":\"f6df2790-d7e0-11ee-ad81-217e54128a4b\",\"type\":\"math\",\"variables\":[{\"id\":\"f8ee0a60-d7e0-11ee-ad81-217e54128a4b\",\"name\":\"packets\",\"field\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\"}],\"script\":\"params.packets*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Packets dropped\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"20b9a420-d7df-11ee-ad81-217e54128a4b\",\"color\":\"rgba(211,96,134,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.capture_loss.gaps\"},{\"id\":\"9a3afce0-d7df-11ee-ad81-217e54128a4b\",\"type\":\"math\",\"variables\":[{\"id\":\"9dece150-d7df-11ee-ad81-217e54128a4b\",\"name\":\"gaps\",\"field\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\"}],\"script\":\"params.gaps*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"ACKS missed\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"filter\":{\"query\":\"event.provider:zeek\",\"language\":\"kuery\"},\"legend_position\":\"right\",\"background_color\":null}}", - "uiStateJSON": "{}", - "description": "Positive values on the y-axis represent observed traffic (packets and ACKs), while negative values represent missing dropped packets and missing ACKs.\n\nThis data is logged by Zeek in stats.log (https://docs.zeek.org/en/master/scripts/policy/misc/stats.zeek.html#type-Stats::Info) and capture_loss.log (https://docs.zeek.org/en/master/scripts/policy/misc/capture-loss.zeek.html#type-CaptureLoss::Info).", + "title": "Last Capture Metric Timestamp by Host", + "visState": "{\"title\":\"Last Capture Metric Timestamp by Host\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"top_hits\",\"params\":{\"field\":\"@timestamp\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Last Metric Timestamp\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"host.name\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Capture Host\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.provider\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":5,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Other\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{\"vis\":{\"sortColumn\":{\"colIndex\":2,\"direction\":\"desc\"}}}", + "description": "", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + "searchSourceJSON": "{\"query\":{\"query\":\"event.kind:metric\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, - "references": [], + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], "migrationVersion": { "visualization": "7.10.0" } }, { - "id": "ba90d2c0-d7e5-11ee-b25e-e793ed358448", + "id": "611a3e20-d7e0-11ee-b25e-e793ed358448", "type": "visualization", "namespaces": [ "default" ], - "updated_at": "2024-03-01T16:06:57.260Z", - "version": "Wzk3MSwxXQ==", + "updated_at": "2024-03-01T18:50:17.404Z", + "version": "Wzk5NywxXQ==", "attributes": { - "title": "Zeek Capture - Host Filter", - "visState": "{\"title\":\"Zeek Capture - Host Filter\",\"type\":\"input_control_vis\",\"aggs\":[],\"params\":{\"controls\":[{\"id\":\"1709309167905\",\"fieldName\":\"host.name\",\"parent\":\"\",\"label\":\"Capture Host\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"dynamicOptions\":true,\"size\":5,\"order\":\"desc\"},\"indexPatternRefName\":\"control_0_index_pattern\"}],\"updateFiltersOnChange\":false,\"useTimeFilter\":false,\"pinFilters\":true}}", + "title": "Zeek and Suricata Capture Measurements ", + "visState": "{\"title\":\"Zeek and Suricata Capture Measurements \",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"32d1fca0-d7e1-11ee-ad81-217e54128a4b\",\"color\":\"rgba(33,150,243,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"32d1fca1-d7e1-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_link\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Zeek: packets seen\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"32d1fca1-d7e1-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"02bbf6a0-d7fb-11ee-a5f1-9ff9da698a18\",\"color\":\"rgba(84,179,153,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"unit\":\"\",\"id\":\"02bbf6a1-d7fb-11ee-a5f1-9ff9da698a18\",\"type\":\"positive_rate\",\"field\":\"suricata.stats.capture.kernel_packets\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Suricata: packets seen\",\"type\":\"timeseries\"},{\"id\":\"e4143600-d7e0-11ee-ad81-217e54128a4b\",\"color\":\"rgba(229,115,115,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_dropped\"},{\"id\":\"f6df2790-d7e0-11ee-ad81-217e54128a4b\",\"type\":\"math\",\"variables\":[{\"id\":\"f8ee0a60-d7e0-11ee-ad81-217e54128a4b\",\"name\":\"packets\",\"field\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\"}],\"script\":\"params.packets*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Zeek: packets dropped\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"20b9a420-d7df-11ee-ad81-217e54128a4b\",\"color\":\"rgba(211,96,134,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.capture_loss.gaps\"},{\"id\":\"9a3afce0-d7df-11ee-ad81-217e54128a4b\",\"type\":\"math\",\"variables\":[{\"id\":\"9dece150-d7df-11ee-ad81-217e54128a4b\",\"name\":\"gaps\",\"field\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\"}],\"script\":\"params.gaps*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Zeek: ACKS missed\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"cad40600-d7fb-11ee-a5f1-9ff9da698a18\",\"color\":\"rgba(255,171,145,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"unit\":\"\",\"id\":\"cad40601-d7fb-11ee-a5f1-9ff9da698a18\",\"type\":\"positive_rate\",\"field\":\"suricata.stats.capture.kernel_drops\"},{\"id\":\"f5352cd0-d7fb-11ee-a5f1-9ff9da698a18\",\"type\":\"math\",\"variables\":[{\"id\":\"f79def70-d7fb-11ee-a5f1-9ff9da698a18\",\"name\":\"packets\",\"field\":\"cad40601-d7fb-11ee-a5f1-9ff9da698a18\"}],\"script\":\"params.packets*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Suricata: packets dropped\",\"type\":\"timeseries\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"filter\":{\"query\":\"(event.provider:zeek OR event.provider:suricata) AND event.kind:metric\",\"language\":\"kuery\"},\"legend_position\":\"right\",\"background_color\":null}}", "uiStateJSON": "{}", - "description": "", + "description": "Positive values on the y-axis represent observed packets while negative values represent missing dropped packets and missing ACKs.\n\nThis data is logged by Zeek in stats.log (https://docs.zeek.org/en/master/scripts/policy/misc/stats.zeek.html#type-Stats::Info) and capture_loss.log (https://docs.zeek.org/en/master/scripts/policy/misc/capture-loss.zeek.html#type-CaptureLoss::Info), and by Suricata (https://docs.suricata.io/en/suricata-6.0.0/performance/statistics.html).", "version": 1, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" } }, - "references": [ - { - "name": "control_0_index_pattern", - "type": "index-pattern", - "id": "malcolm_beats_*" - } - ], + "references": [], "migrationVersion": { "visualization": "7.10.0" } @@ -162,6 +177,29 @@ "visualization": "7.10.0" } }, + { + "id": "8051d2c0-d7e6-11ee-b25e-e793ed358448", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T16:12:29.036Z", + "version": "Wzk3NCwxXQ==", + "attributes": { + "title": "Zeek Stats - Capture Loss", + "visState": "{\"title\":\"Zeek Stats - Capture Loss\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_dropped\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Packets Dropped\"},{\"id\":\"bd4560e0-d7e4-11ee-ad81-217e54128a4b\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"bd4560e1-d7e4-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.capture_loss.gaps\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"ACKs Missed\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:zeek AND event.dataset:(stats OR capture_loss)\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + }, { "id": "991158b0-d7d9-11ee-9ed3-e7afff29e59a", "type": "search", @@ -200,46 +238,39 @@ } }, { - "id": "abcfca50-d7dd-11ee-b25e-e793ed358448", + "id": "cc976b70-d7fc-11ee-b25e-e793ed358448", "type": "visualization", "namespaces": [ "default" ], - "updated_at": "2024-03-01T15:09:16.533Z", - "version": "Wzk1OSwxXQ==", + "updated_at": "2024-03-01T18:56:30.009Z", + "version": "WzEwMDksMV0=", "attributes": { - "title": "Zeek Analyzer Messages", - "visState": "{\"title\":\"Zeek Analyzer Messages\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.cause\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Cause\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.analyzer_kind\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Class\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.analyzer_name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Analyzer\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "title": "Suricata Stats - Packets and Bytes", + "visState": "{\"title\":\"Suricata Stats - Packets and Bytes\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"unit\":\"\",\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"top_hit\",\"field\":\"suricata.stats.capture.kernel_packets\",\"order_by\":\"@timestamp\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Total Packets Seen\"},{\"id\":\"bd4560e0-d7e4-11ee-ad81-217e54128a4b\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"bd4560e1-d7e4-11ee-ad81-217e54128a4b\",\"type\":\"top_hit\",\"field\":\"suricata.stats.decoder.bytes\",\"order_by\":\"@timestamp\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Total Bytes Seen\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:suricata AND event.kind:metric\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", "uiStateJSON": "{}", "description": "", "version": 1, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" - }, - "savedSearchRefName": "search_0" - }, - "references": [ - { - "name": "search_0", - "type": "search", - "id": "351ef380-d7d9-11ee-9ed3-e7afff29e59a" } - ], + }, + "references": [], "migrationVersion": { "visualization": "7.10.0" } }, { - "id": "8051d2c0-d7e6-11ee-b25e-e793ed358448", + "id": "9833e6a0-d7fd-11ee-b25e-e793ed358448", "type": "visualization", "namespaces": [ "default" ], - "updated_at": "2024-03-01T16:12:29.036Z", - "version": "Wzk3NCwxXQ==", + "updated_at": "2024-03-01T19:02:50.252Z", + "version": "WzEwNzksMV0=", "attributes": { - "title": "Zeek Stats - Capture Loss", - "visState": "{\"title\":\"Zeek Stats - Capture Loss\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_dropped\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Packets Dropped\"},{\"id\":\"bd4560e0-d7e4-11ee-ad81-217e54128a4b\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"bd4560e1-d7e4-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.capture_loss.gaps\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"ACKs Missed\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:zeek AND event.dataset:(stats OR capture_loss)\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", + "title": "Suricata Stats - Capture Loss", + "visState": "{\"title\":\"Suricata Stats - Capture Loss\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"top_hit\",\"field\":\"suricata.stats.capture.kernel_drops\",\"order_by\":\"@timestamp\"},{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"ffb34320-d7fd-11ee-a5f1-9ff9da698a18\",\"type\":\"top_hit\",\"field\":\"suricata.stats.tcp.segment_memcap_drop\",\"order_by\":\"@timestamp\"},{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"0e962510-d7fe-11ee-a5f1-9ff9da698a18\",\"type\":\"top_hit\",\"field\":\"suricata.stats.tcp.ssn_memcap_drop\",\"order_by\":\"@timestamp\"},{\"id\":\"1d60c410-d7fe-11ee-a5f1-9ff9da698a18\",\"type\":\"math\",\"variables\":[{\"id\":\"21851960-d7fe-11ee-a5f1-9ff9da698a18\",\"name\":\"kernel\",\"field\":\"61ca57f2-469d-11e7-af02-69e470af7417\"},{\"id\":\"25a01fe0-d7fe-11ee-a5f1-9ff9da698a18\",\"name\":\"segment\",\"field\":\"ffb34320-d7fd-11ee-a5f1-9ff9da698a18\"},{\"id\":\"2b81b590-d7fe-11ee-a5f1-9ff9da698a18\",\"name\":\"session\",\"field\":\"0e962510-d7fe-11ee-a5f1-9ff9da698a18\"}],\"script\":\"params.kernel+params.segment+params.session\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Packets Dropped\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:suricata AND event.kind:metric\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", "uiStateJSON": "{}", "description": "", "version": 1, @@ -253,21 +284,21 @@ } }, { - "id": "65b7efe0-d7e5-11ee-b25e-e793ed358448", + "id": "6b7b9a40-faa1-11e6-86b1-cd7735ff7e23-ecs", "type": "visualization", "namespaces": [ "default" ], - "updated_at": "2024-03-01T16:04:34.910Z", - "version": "Wzk3MCwxXQ==", + "updated_at": "2024-03-01T15:01:49.331Z", + "version": "WzkxNywxXQ==", "attributes": { - "title": "Zeek Stats - Files", - "visState": "{\"title\":\"Zeek Stats - Files\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"zeek.stats.files\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Files Seen\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"400f2100-d7e5-11ee-ad81-217e54128a4b\"}],\"time_range_mode\":\"entire_time_range\",\"filter\":{\"query\":\"event.provider:zeek AND event.dataset:stats\",\"language\":\"kuery\"}}}", + "title": "Network Traffic (Packets)", + "visState": "{\"title\":\"Network Traffic (Packets)\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"axis_scale\":\"normal\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"filter\":{\"language\":\"lucene\",\"query\":\"\"},\"id\":\"da1046f0-faa0-11e6-86b1-cd7735ff7e23\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"auto\",\"isModelInvalid\":false,\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"rgba(0,133,255,1)\",\"fill\":\"1\",\"formatter\":\"'0a'\",\"id\":\"49931900-ebf3-11ec-a401-f5db2d59e6af\",\"label\":\"Inbound\",\"line_width\":1,\"metrics\":[{\"unit\":\"1s\",\"id\":\"49931901-ebf3-11ec-a401-f5db2d59e6af\",\"type\":\"positive_rate\",\"field\":\"miscbeat.network.packets.rx\"}],\"point_size\":1,\"separate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"type\":\"timeseries\",\"terms_field\":\"miscbeat.network.interface\",\"terms_size\":\"3\",\"terms_order_by\":\"_count\",\"value_template\":\"{{value}}/s\",\"split_color_mode\":\"gradient\"},{\"id\":\"75fba890-ebf3-11ec-a401-f5db2d59e6af\",\"color\":\"rgba(13,212,26,1)\",\"split_mode\":\"terms\",\"metrics\":[{\"unit\":\"1s\",\"id\":\"75fba891-ebf3-11ec-a401-f5db2d59e6af\",\"type\":\"positive_rate\",\"field\":\"miscbeat.network.packets.tx\"},{\"id\":\"96daba60-ebf3-11ec-a401-f5db2d59e6af\",\"type\":\"math\",\"variables\":[{\"id\":\"98e138c0-ebf3-11ec-a401-f5db2d59e6af\",\"name\":\"rate\",\"field\":\"75fba891-ebf3-11ec-a401-f5db2d59e6af\"}],\"script\":\"params.rate*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"'0a'\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":\"1\",\"stacked\":\"none\",\"label\":\"Outbound\",\"type\":\"timeseries\",\"terms_size\":\"3\",\"terms_field\":\"miscbeat.network.interface\",\"terms_order_by\":\"_count\",\"split_color_mode\":\"gradient\",\"value_template\":\"{{value}}/s\"}],\"show_grid\":1,\"show_legend\":1,\"time_field\":\"@timestamp\",\"tooltip_mode\":\"show_all\",\"type\":\"timeseries\"}}", "uiStateJSON": "{}", "description": "", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" } }, "references": [], @@ -317,6 +348,77 @@ "search": "7.9.3" } }, + { + "id": "b0aabf40-d7fa-11ee-b25e-e793ed358448", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T18:37:00.084Z", + "version": "Wzk5NCwxXQ==", + "attributes": { + "title": "Packet Capture - Suricata Stats", + "description": "", + "hits": 0, + "columns": [ + "host.name", + "suricata.stats.capture.kernel_packets", + "suricata.stats.capture.kernel_drops", + "suricata.stats.capture.errors", + "suricata.stats.decoder.bytes", + "suricata.stats.decoder.ethernet", + "suricata.stats.decoder.ipv4", + "suricata.stats.decoder.ipv6", + "suricata.stats.detect.engines.rules_loaded", + "suricata.stats.detect.alert" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.provider:suricata AND event.kind:metric\",\"language\":\"lucene\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30m\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + }, + { + "id": "abcfca50-d7dd-11ee-b25e-e793ed358448", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T19:07:58.499Z", + "version": "WzEwOTEsMV0=", + "attributes": { + "title": "Zeek Analyzer Messages", + "visState": "{\"title\":\"Zeek Analyzer Messages\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.cause\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Cause\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.analyzer_kind\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Class\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.analyzer_name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Analyzer\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "351ef380-d7d9-11ee-9ed3-e7afff29e59a" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, { "id": "351ef380-d7d9-11ee-9ed3-e7afff29e59a", "type": "search", @@ -422,37 +524,26 @@ } }, { - "id": "e7047c50-d7d9-11ee-9ed3-e7afff29e59a", - "type": "search", + "id": "089b85d0-1b16-11e7-b09e-037021c4f8df-ecs", + "type": "visualization", "namespaces": [ "default" ], - "updated_at": "2024-03-01T15:01:44.258Z", - "version": "Wzg4NiwxXQ==", + "updated_at": "2024-03-01T15:01:49.331Z", + "version": "WzkxOCwxXQ==", "attributes": { - "title": "Packet Capture - Zeek cluster.log", + "title": "Network Traffic (Bytes)", + "visState": "{\"title\":\"Network Traffic (Bytes)\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"axis_scale\":\"normal\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"filter\":{\"language\":\"lucene\",\"query\":\"\"},\"id\":\"da1046f0-faa0-11e6-86b1-cd7735ff7e23\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"auto\",\"isModelInvalid\":false,\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"rgba(0,133,255,1)\",\"fill\":\"1\",\"formatter\":\"bytes\",\"id\":\"6d8b8ab0-ebf1-11ec-a401-f5db2d59e6af\",\"line_width\":1,\"metrics\":[{\"unit\":\"1s\",\"id\":\"6d8b8ab1-ebf1-11ec-a401-f5db2d59e6af\",\"type\":\"positive_rate\",\"field\":\"miscbeat.network.bytes.rx\"}],\"point_size\":1,\"separate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"label\":\"Inbound\",\"type\":\"timeseries\",\"terms_field\":\"miscbeat.network.interface\",\"terms_size\":\"3\",\"terms_order_by\":\"_key\",\"value_template\":\"{{value}}/s\",\"split_color_mode\":\"gradient\"},{\"id\":\"b5977de0-ebf2-11ec-a401-f5db2d59e6af\",\"color\":\"rgba(13,212,26,1)\",\"split_mode\":\"terms\",\"metrics\":[{\"unit\":\"1s\",\"id\":\"b5977de1-ebf2-11ec-a401-f5db2d59e6af\",\"type\":\"positive_rate\",\"field\":\"miscbeat.network.bytes.tx\"},{\"id\":\"cdfb1540-ebf2-11ec-a401-f5db2d59e6af\",\"type\":\"math\",\"variables\":[{\"id\":\"d1b9caf0-ebf2-11ec-a401-f5db2d59e6af\",\"name\":\"rate\",\"field\":\"b5977de1-ebf2-11ec-a401-f5db2d59e6af\"}],\"script\":\"params.rate*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":\"1\",\"stacked\":\"none\",\"label\":\"Outbound\",\"split_color_mode\":\"gradient\",\"type\":\"timeseries\",\"terms_size\":\"3\",\"terms_order_by\":\"_key\",\"terms_field\":\"miscbeat.network.interface\",\"value_template\":\"{{value}}/s\"}],\"show_grid\":1,\"show_legend\":1,\"time_field\":\"@timestamp\",\"tooltip_mode\":\"show_all\",\"type\":\"timeseries\"}}", + "uiStateJSON": "{}", "description": "", - "hits": 0, - "columns": [ - "host.name", - "zeek.cluster.node", - "zeek.cluster.message" - ], - "sort": [], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"event.provider:zeek and event.dataset:cluster\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30d\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" } }, - "references": [ - { - "name": "kibanaSavedObjectMeta.searchSourceJSON.index", - "type": "index-pattern", - "id": "malcolm_beats_*" - } - ], + "references": [], "migrationVersion": { - "search": "7.9.3" + "visualization": "7.10.0" } } ] From 8a13ba2c02b02411e7c4c0855f5cda3e3742e4bf Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 1 Mar 2024 14:41:28 -0700 Subject: [PATCH 67/79] Work in progress for idaholab/Malcolm#414, windows event logs dashboard for zeek --- .../79202ee0-d811-11ee-820d-dd9fd73a3921.json | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json diff --git a/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json b/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json new file mode 100644 index 000000000..68dbbe7d7 --- /dev/null +++ b/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json @@ -0,0 +1,249 @@ +{ + "version": "2.12.0", + "objects": [ + { + "id": "79202ee0-d811-11ee-820d-dd9fd73a3921", + "type": "dashboard", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T21:33:15.446Z", + "version": "Wzk5MywxXQ==", + "attributes": { + "title": "Windows Events", + "hits": 0, + "description": "", + "panelsJSON": "[{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":18,\"h\":15,\"i\":\"346bb696-5fa2-4504-a1d8-5a6f51244c7b\"},\"panelIndex\":\"346bb696-5fa2-4504-a1d8-5a6f51244c7b\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":2,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_0\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":18,\"y\":0,\"w\":30,\"h\":15,\"i\":\"9c39d8b3-ad8c-4247-b97f-9736e469c988\"},\"panelIndex\":\"9c39d8b3-ad8c-4247-b97f-9736e469c988\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_1\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":15,\"w\":30,\"h\":36,\"i\":\"bccfb126-a864-4c11-a8c7-a9a1286c8f0f\"},\"panelIndex\":\"bccfb126-a864-4c11-a8c7-a9a1286c8f0f\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":30,\"y\":15,\"w\":18,\"h\":15,\"i\":\"bc116b54-f251-4e77-833c-c557b5d5c1d7\"},\"panelIndex\":\"bc116b54-f251-4e77-833c-c557b5d5c1d7\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_3\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":30,\"y\":30,\"w\":18,\"h\":21,\"i\":\"5566d8f7-5c04-4f92-acf3-5af407b28b0d\"},\"panelIndex\":\"5566d8f7-5c04-4f92-acf3-5af407b28b0d\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_4\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":51,\"w\":48,\"h\":30,\"i\":\"49f38efc-1ab3-4e38-96e5-b0458c026491\"},\"panelIndex\":\"49f38efc-1ab3-4e38-96e5-b0458c026491\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"}]", + "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", + "version": 1, + "timeRestore": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + } + }, + "references": [ + { + "name": "panel_0", + "type": "visualization", + "id": "0100b010-d811-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_1", + "type": "visualization", + "id": "6a4710f0-d811-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_2", + "type": "visualization", + "id": "db80a970-d811-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_3", + "type": "visualization", + "id": "4d51c700-d812-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_4", + "type": "visualization", + "id": "f7fcb9d0-d812-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_5", + "type": "search", + "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "dashboard": "7.9.3" + } + }, + { + "id": "0100b010-d811-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T21:16:43.792Z", + "version": "Wzk4MCwxXQ==", + "attributes": { + "title": "Windows Events by Host", + "visState": "{\"title\":\"Windows Events by Host\",\"type\":\"table\",\"aggs\":[{\"id\":\"4\",\"enabled\":true,\"type\":\"top_hits\",\"params\":{\"field\":\"@timestamp\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Last Log\"},\"schema\":\"metric\"},{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"host.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Host Forwarder\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.winlog.Computer\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Computer Name\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "6a4710f0-d811-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T21:20:23.112Z", + "version": "Wzk4MywxXQ==", + "attributes": { + "title": "Windows Events over Time", + "visState": "{\"title\":\"Windows Events over Time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now/d\",\"to\":\"now/d\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.dataset\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "db80a970-d811-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T21:28:23.050Z", + "version": "Wzk4OCwxXQ==", + "attributes": { + "title": "Windows Event Provider", + "visState": "{\"title\":\"Windows Event Provider\",\"type\":\"horizontal_bar\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.winlog.ProviderName\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Event Provider\"},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.winlog.ProviderName\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Event Provider\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":50},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":75,\"filter\":true,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":true},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "4d51c700-d812-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T21:26:30.332Z", + "version": "Wzk4NywxXQ==", + "attributes": { + "title": "Windows Event Results", + "visState": "{\"title\":\"Windows Event Results\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.result\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Event Result\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "f7fcb9d0-d812-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T21:30:47.661Z", + "version": "Wzk4OSwxXQ==", + "attributes": { + "title": "Windows Event Insertion Strings", + "visState": "{\"title\":\"Windows Event Insertion Strings\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.winlog.StringInserts\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":250,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Insertion Strings\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T20:24:45.212Z", + "version": "Wzk3OCwxXQ==", + "attributes": { + "title": "Windows Event Logs", + "description": "", + "hits": 0, + "columns": [ + "host.name", + "miscbeat.winlog.Computer", + "event.dataset", + "event.id", + "miscbeat.winlog.ProviderName", + "event.result", + "event.original" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.module:(winlog OR winevtlog)\",\"language\":\"lucene\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + } + ] +} \ No newline at end of file From 6b332e7f3919153e7e9e244f5078b9f64a786132 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 1 Mar 2024 15:01:11 -0700 Subject: [PATCH 68/79] Work in progress for idaholab/Malcolm#414, windows event logs dashboard for zeek --- .../79202ee0-d811-11ee-820d-dd9fd73a3921.json | 108 +++++++++++++++++- 1 file changed, 104 insertions(+), 4 deletions(-) diff --git a/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json b/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json index 68dbbe7d7..779cbf43d 100644 --- a/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json +++ b/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json @@ -7,18 +7,18 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T21:33:15.446Z", - "version": "Wzk5MywxXQ==", + "updated_at": "2024-03-01T22:00:38.756Z", + "version": "WzEwOTUsMV0=", "attributes": { "title": "Windows Events", "hits": 0, "description": "", - "panelsJSON": "[{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":18,\"h\":15,\"i\":\"346bb696-5fa2-4504-a1d8-5a6f51244c7b\"},\"panelIndex\":\"346bb696-5fa2-4504-a1d8-5a6f51244c7b\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":2,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_0\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":18,\"y\":0,\"w\":30,\"h\":15,\"i\":\"9c39d8b3-ad8c-4247-b97f-9736e469c988\"},\"panelIndex\":\"9c39d8b3-ad8c-4247-b97f-9736e469c988\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_1\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":15,\"w\":30,\"h\":36,\"i\":\"bccfb126-a864-4c11-a8c7-a9a1286c8f0f\"},\"panelIndex\":\"bccfb126-a864-4c11-a8c7-a9a1286c8f0f\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":30,\"y\":15,\"w\":18,\"h\":15,\"i\":\"bc116b54-f251-4e77-833c-c557b5d5c1d7\"},\"panelIndex\":\"bc116b54-f251-4e77-833c-c557b5d5c1d7\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_3\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":30,\"y\":30,\"w\":18,\"h\":21,\"i\":\"5566d8f7-5c04-4f92-acf3-5af407b28b0d\"},\"panelIndex\":\"5566d8f7-5c04-4f92-acf3-5af407b28b0d\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_4\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":51,\"w\":48,\"h\":30,\"i\":\"49f38efc-1ab3-4e38-96e5-b0458c026491\"},\"panelIndex\":\"49f38efc-1ab3-4e38-96e5-b0458c026491\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"}]", + "panelsJSON": "[{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"346bb696-5fa2-4504-a1d8-5a6f51244c7b\",\"w\":18,\"x\":0,\"y\":0},\"panelIndex\":\"346bb696-5fa2-4504-a1d8-5a6f51244c7b\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":2,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_0\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"9c39d8b3-ad8c-4247-b97f-9736e469c988\",\"w\":30,\"x\":18,\"y\":0},\"panelIndex\":\"9c39d8b3-ad8c-4247-b97f-9736e469c988\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_1\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":36,\"i\":\"bccfb126-a864-4c11-a8c7-a9a1286c8f0f\",\"w\":30,\"x\":0,\"y\":15},\"panelIndex\":\"bccfb126-a864-4c11-a8c7-a9a1286c8f0f\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"bc116b54-f251-4e77-833c-c557b5d5c1d7\",\"w\":18,\"x\":30,\"y\":15},\"panelIndex\":\"bc116b54-f251-4e77-833c-c557b5d5c1d7\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_3\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":21,\"i\":\"5566d8f7-5c04-4f92-acf3-5af407b28b0d\",\"w\":18,\"x\":30,\"y\":30},\"panelIndex\":\"5566d8f7-5c04-4f92-acf3-5af407b28b0d\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":0,\"direction\":\"asc\"}}},\"panelRefName\":\"panel_4\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":30,\"i\":\"49f38efc-1ab3-4e38-96e5-b0458c026491\",\"w\":48,\"x\":0,\"y\":66},\"panelIndex\":\"49f38efc-1ab3-4e38-96e5-b0458c026491\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":25,\"i\":\"0249a576-f162-4bf9-aa2d-bf85552665c7\",\"w\":48,\"x\":0,\"y\":96},\"panelIndex\":\"0249a576-f162-4bf9-aa2d-bf85552665c7\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"65476182-297e-4bbf-a7af-b0ff0584eeb0\",\"w\":24,\"x\":0,\"y\":51},\"panelIndex\":\"65476182-297e-4bbf-a7af-b0ff0584eeb0\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_7\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":24,\"y\":51,\"w\":24,\"h\":15,\"i\":\"0bf80b0d-91f3-47ca-b2a3-ea6844c9727a\"},\"panelIndex\":\"0bf80b0d-91f3-47ca-b2a3-ea6844c9727a\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_8\"}]", "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}" } }, "references": [ @@ -51,6 +51,21 @@ "name": "panel_5", "type": "search", "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_6", + "type": "search", + "id": "3770db80-d815-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_7", + "type": "visualization", + "id": "7e9a8500-d816-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_8", + "type": "visualization", + "id": "008fb710-d817-11ee-820d-dd9fd73a3921" } ], "migrationVersion": { @@ -244,6 +259,91 @@ "migrationVersion": { "search": "7.9.3" } + }, + { + "id": "3770db80-d815-11ee-820d-dd9fd73a3921", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T21:46:53.112Z", + "version": "WzEwMDQsMV0=", + "attributes": { + "title": "Windows Resource Utilization", + "description": "", + "hits": 0, + "columns": [ + "host.name", + "miscbeat.winstat.cpu_utilization", + "miscbeat.winstat.physical_total", + "miscbeat.winstat.physical_used", + "miscbeat.winstat.physical_available", + "miscbeat.winstat.processes", + "miscbeat.winstat.threads", + "miscbeat.winstat.handles" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.module:winstat\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + }, + { + "id": "7e9a8500-d816-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T21:56:02.000Z", + "version": "WzEwNjEsMV0=", + "attributes": { + "title": "Windows RAM Usage", + "visState": "{\"title\":\"Windows RAM Usage\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"gauge\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"terms\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"top_hit\",\"field\":\"miscbeat.winstat.physical_used\",\"order_by\":\"@timestamp\"},{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"e4b6eb90-d815-11ee-a6da-0f56a6d37163\",\"type\":\"top_hit\",\"field\":\"miscbeat.winstat.physical_total\",\"order_by\":\"@timestamp\"},{\"id\":\"fe21e0d0-d815-11ee-a6da-0f56a6d37163\",\"type\":\"math\",\"variables\":[{\"id\":\"01e57880-d816-11ee-a6da-0f56a6d37163\",\"name\":\"used\",\"field\":\"61ca57f2-469d-11e7-af02-69e470af7417\"},{\"id\":\"082756a0-d816-11ee-a6da-0f56a6d37163\",\"name\":\"total\",\"field\":\"e4b6eb90-d815-11ee-a6da-0f56a6d37163\"}],\"script\":\"(params.used/params.total)*100\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"00.\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"terms_field\":\"host.name\",\"label\":\"RAM Usage\",\"terms_order_by\":\"_count\",\"value_template\":\"{{value}}%\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"gauge_color_rules\":[{\"id\":\"8db33970-d815-11ee-a6da-0f56a6d37163\"}],\"gauge_width\":10,\"gauge_inner_width\":10,\"gauge_style\":\"half\",\"filter\":{\"query\":\"event.module:winstat\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\",\"gauge_max\":\"100\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "008fb710-d817-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T21:59:40.033Z", + "version": "WzEwOTIsMV0=", + "attributes": { + "title": "Windows CPU Usage", + "visState": "{\"title\":\"Windows CPU Usage\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"gauge\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"terms\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"size\":1,\"agg_with\":\"noop\",\"order\":\"desc\",\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"top_hit\",\"field\":\"miscbeat.winstat.cpu_utilization\",\"order_by\":\"@timestamp\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"00\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"CPU Usage\",\"value_template\":\"{{value}}%\",\"filter\":{\"query\":\"event.module:winstat\",\"language\":\"kuery\"},\"terms_field\":\"host.name\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"gauge_color_rules\":[{\"id\":\"a50173c0-d816-11ee-a6da-0f56a6d37163\"}],\"gauge_width\":10,\"gauge_inner_width\":10,\"gauge_style\":\"half\",\"gauge_max\":\"100\",\"time_range_mode\":\"entire_time_range\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } } ] } \ No newline at end of file From 921fe4d13a104e5d7339cdaa7b78871ec6fbd424 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 1 Mar 2024 15:16:19 -0700 Subject: [PATCH 69/79] Work in progress for idaholab/Malcolm#414, windows kernel messages dashboard for zeek --- .../3768ef70-d819-11ee-820d-dd9fd73a3921.json | 141 ++++++++++++++++++ .../79202ee0-d811-11ee-820d-dd9fd73a3921.json | 10 +- 2 files changed, 146 insertions(+), 5 deletions(-) create mode 100644 dashboards/dashboards/beats/3768ef70-d819-11ee-820d-dd9fd73a3921.json diff --git a/dashboards/dashboards/beats/3768ef70-d819-11ee-820d-dd9fd73a3921.json b/dashboards/dashboards/beats/3768ef70-d819-11ee-820d-dd9fd73a3921.json new file mode 100644 index 000000000..cabf42f71 --- /dev/null +++ b/dashboards/dashboards/beats/3768ef70-d819-11ee-820d-dd9fd73a3921.json @@ -0,0 +1,141 @@ +{ + "version": "2.12.0", + "objects": [ + { + "id": "3768ef70-d819-11ee-820d-dd9fd73a3921", + "type": "dashboard", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T22:15:31.047Z", + "version": "WzExMTEsMV0=", + "attributes": { + "title": "Linux Kernel Messages", + "hits": 0, + "description": "", + "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"h\":16,\"i\":\"1c0ec894-1fd1-4a1a-8aa1-730e651ecca8\",\"w\":14,\"x\":0,\"y\":0},\"panelIndex\":\"1c0ec894-1fd1-4a1a-8aa1-730e651ecca8\",\"version\":\"2.12.0\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":16,\"i\":\"d395e5d6-8064-4226-b4bd-7c21db0d6cf5\",\"w\":34,\"x\":14,\"y\":0},\"panelIndex\":\"d395e5d6-8064-4226-b4bd-7c21db0d6cf5\",\"version\":\"2.12.0\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":19,\"i\":\"4178095b-48cb-4ce8-b79c-eea288285dd5\",\"w\":48,\"x\":0,\"y\":16},\"panelIndex\":\"4178095b-48cb-4ce8-b79c-eea288285dd5\",\"version\":\"2.12.0\",\"panelRefName\":\"panel_2\"}]", + "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", + "version": 1, + "timeRestore": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}" + } + }, + "references": [ + { + "name": "panel_0", + "type": "visualization", + "id": "822595a0-d818-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_1", + "type": "visualization", + "id": "ab235be0-d818-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_2", + "type": "search", + "id": "27ac44c0-d818-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "dashboard": "7.9.3" + } + }, + { + "id": "822595a0-d818-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T22:10:26.938Z", + "version": "WzExMDgsMV0=", + "attributes": { + "title": "Linux Kernel Messages by Host", + "visState": "{\"title\":\"Linux Kernel Messages by Host\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"host.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Host\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"top_hits\",\"params\":{\"field\":\"@timestamp\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Last Kernel Message\"},\"schema\":\"metric\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "27ac44c0-d818-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "ab235be0-d818-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T22:14:59.550Z", + "version": "WzExMTAsMV0=", + "attributes": { + "title": "Linux Kernel Messages over Time", + "visState": "{\"title\":\"Linux Kernel Messages over Time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{\"customLabel\":\"Message Count\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now/d\",\"to\":\"now/d\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Messages over Time by Priority\"},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.kmsg.priority\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Priority\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Message Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Message Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "27ac44c0-d818-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "27ac44c0-d818-11ee-820d-dd9fd73a3921", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T22:07:55.148Z", + "version": "WzExMDUsMV0=", + "attributes": { + "title": "Linux Kernel Messages", + "description": "", + "hits": 0, + "columns": [ + "host.name", + "miscbeat.kmsg.sequence", + "miscbeat.kmsg.priority", + "event.original" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.module:kmsg\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30m\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + } + ] +} \ No newline at end of file diff --git a/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json b/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json index 779cbf43d..7312c147e 100644 --- a/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json +++ b/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json @@ -7,13 +7,13 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T22:00:38.756Z", - "version": "WzEwOTUsMV0=", + "updated_at": "2024-03-01T22:03:46.831Z", + "version": "WzEwOTgsMV0=", "attributes": { "title": "Windows Events", "hits": 0, "description": "", - "panelsJSON": "[{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"346bb696-5fa2-4504-a1d8-5a6f51244c7b\",\"w\":18,\"x\":0,\"y\":0},\"panelIndex\":\"346bb696-5fa2-4504-a1d8-5a6f51244c7b\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":2,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_0\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"9c39d8b3-ad8c-4247-b97f-9736e469c988\",\"w\":30,\"x\":18,\"y\":0},\"panelIndex\":\"9c39d8b3-ad8c-4247-b97f-9736e469c988\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_1\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":36,\"i\":\"bccfb126-a864-4c11-a8c7-a9a1286c8f0f\",\"w\":30,\"x\":0,\"y\":15},\"panelIndex\":\"bccfb126-a864-4c11-a8c7-a9a1286c8f0f\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"bc116b54-f251-4e77-833c-c557b5d5c1d7\",\"w\":18,\"x\":30,\"y\":15},\"panelIndex\":\"bc116b54-f251-4e77-833c-c557b5d5c1d7\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_3\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":21,\"i\":\"5566d8f7-5c04-4f92-acf3-5af407b28b0d\",\"w\":18,\"x\":30,\"y\":30},\"panelIndex\":\"5566d8f7-5c04-4f92-acf3-5af407b28b0d\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":0,\"direction\":\"asc\"}}},\"panelRefName\":\"panel_4\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":30,\"i\":\"49f38efc-1ab3-4e38-96e5-b0458c026491\",\"w\":48,\"x\":0,\"y\":66},\"panelIndex\":\"49f38efc-1ab3-4e38-96e5-b0458c026491\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":25,\"i\":\"0249a576-f162-4bf9-aa2d-bf85552665c7\",\"w\":48,\"x\":0,\"y\":96},\"panelIndex\":\"0249a576-f162-4bf9-aa2d-bf85552665c7\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"65476182-297e-4bbf-a7af-b0ff0584eeb0\",\"w\":24,\"x\":0,\"y\":51},\"panelIndex\":\"65476182-297e-4bbf-a7af-b0ff0584eeb0\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_7\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":24,\"y\":51,\"w\":24,\"h\":15,\"i\":\"0bf80b0d-91f3-47ca-b2a3-ea6844c9727a\"},\"panelIndex\":\"0bf80b0d-91f3-47ca-b2a3-ea6844c9727a\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_8\"}]", + "panelsJSON": "[{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"346bb696-5fa2-4504-a1d8-5a6f51244c7b\",\"w\":18,\"x\":0,\"y\":0},\"panelIndex\":\"346bb696-5fa2-4504-a1d8-5a6f51244c7b\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":2,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_0\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"9c39d8b3-ad8c-4247-b97f-9736e469c988\",\"w\":30,\"x\":18,\"y\":0},\"panelIndex\":\"9c39d8b3-ad8c-4247-b97f-9736e469c988\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_1\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":36,\"i\":\"bccfb126-a864-4c11-a8c7-a9a1286c8f0f\",\"w\":30,\"x\":0,\"y\":15},\"panelIndex\":\"bccfb126-a864-4c11-a8c7-a9a1286c8f0f\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"bc116b54-f251-4e77-833c-c557b5d5c1d7\",\"w\":18,\"x\":30,\"y\":15},\"panelIndex\":\"bc116b54-f251-4e77-833c-c557b5d5c1d7\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}},\"table\":null},\"panelRefName\":\"panel_3\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":21,\"i\":\"5566d8f7-5c04-4f92-acf3-5af407b28b0d\",\"w\":18,\"x\":30,\"y\":30},\"panelIndex\":\"5566d8f7-5c04-4f92-acf3-5af407b28b0d\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_4\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":30,\"i\":\"49f38efc-1ab3-4e38-96e5-b0458c026491\",\"w\":48,\"x\":0,\"y\":66},\"panelIndex\":\"49f38efc-1ab3-4e38-96e5-b0458c026491\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":25,\"i\":\"0249a576-f162-4bf9-aa2d-bf85552665c7\",\"w\":48,\"x\":0,\"y\":96},\"panelIndex\":\"0249a576-f162-4bf9-aa2d-bf85552665c7\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"65476182-297e-4bbf-a7af-b0ff0584eeb0\",\"w\":24,\"x\":0,\"y\":51},\"panelIndex\":\"65476182-297e-4bbf-a7af-b0ff0584eeb0\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_7\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":24,\"y\":51,\"w\":24,\"h\":15,\"i\":\"0bf80b0d-91f3-47ca-b2a3-ea6844c9727a\"},\"panelIndex\":\"0bf80b0d-91f3-47ca-b2a3-ea6844c9727a\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_8\"}]", "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", "version": 1, "timeRestore": false, @@ -246,7 +246,7 @@ "sort": [], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"event.module:(winlog OR winevtlog)\",\"language\":\"lucene\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"query\":{\"query\":\"event.module:(winlog OR winevtlog)\",\"language\":\"lucene\"},\"highlightAll\":false\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ @@ -285,7 +285,7 @@ "sort": [], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"event.module:winstat\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"query\":{\"query\":\"event.module:winstat\",\"language\":\"kuery\"},\"highlightAll\":false\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ From 57a2bf39a31f402902c5dfa650b9de4dd978eb06 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 1 Mar 2024 15:20:30 -0700 Subject: [PATCH 70/79] Work in progress for idaholab/Malcolm#414, additional resource/statistics dashboards --- .../dashboards/152f29dc-51a2-4f53-93e9-6e92765567b8.json | 2 +- .../beats/0d4955f0-eb25-11ec-a6d4-b3526526c2c7.json | 2 +- .../beats/3768ef70-d819-11ee-820d-dd9fd73a3921.json | 2 +- .../beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json | 2 +- .../beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json | 2 +- .../beats/7a7e0a60-e8e8-11ec-b9d4-4569bb965430.json | 2 +- .../beats/903f42c0-f634-11ec-828d-2fb7a4a26e1f.json | 4 ++-- dashboards/dashboards/beats/Filebeat-nginx-logs.json | 6 +++--- dashboards/dashboards/beats/Filebeat-nginx-overview.json | 4 ++-- dashboards/dashboards/beats/Metricbeat-host-overview.json | 2 +- dashboards/dashboards/beats/Metricbeat-system-overview.json | 2 +- .../beats/f6600310-9943-11ee-a029-e973f4774355.json | 2 +- 12 files changed, 16 insertions(+), 16 deletions(-) diff --git a/dashboards/dashboards/152f29dc-51a2-4f53-93e9-6e92765567b8.json b/dashboards/dashboards/152f29dc-51a2-4f53-93e9-6e92765567b8.json index 9ccb3d124..3fe16806b 100644 --- a/dashboards/dashboards/152f29dc-51a2-4f53-93e9-6e92765567b8.json +++ b/dashboards/dashboards/152f29dc-51a2-4f53-93e9-6e92765567b8.json @@ -710,7 +710,7 @@ ], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"event.dataset:modbus_read_device_identification\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"event.dataset:modbus_read_device_identification\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ diff --git a/dashboards/dashboards/beats/0d4955f0-eb25-11ec-a6d4-b3526526c2c7.json b/dashboards/dashboards/beats/0d4955f0-eb25-11ec-a6d4-b3526526c2c7.json index 2903b4b40..983bfdaa0 100644 --- a/dashboards/dashboards/beats/0d4955f0-eb25-11ec-a6d4-b3526526c2c7.json +++ b/dashboards/dashboards/beats/0d4955f0-eb25-11ec-a6d4-b3526526c2c7.json @@ -191,7 +191,7 @@ "sort": [], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"event.module:thermal\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"event.module:thermal\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ diff --git a/dashboards/dashboards/beats/3768ef70-d819-11ee-820d-dd9fd73a3921.json b/dashboards/dashboards/beats/3768ef70-d819-11ee-820d-dd9fd73a3921.json index cabf42f71..d7c991a76 100644 --- a/dashboards/dashboards/beats/3768ef70-d819-11ee-820d-dd9fd73a3921.json +++ b/dashboards/dashboards/beats/3768ef70-d819-11ee-820d-dd9fd73a3921.json @@ -12,7 +12,7 @@ "attributes": { "title": "Linux Kernel Messages", "hits": 0, - "description": "", + "description": "Linux kernel messages (kmsg), including from Malcolm sensors and aggregators", "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"h\":16,\"i\":\"1c0ec894-1fd1-4a1a-8aa1-730e651ecca8\",\"w\":14,\"x\":0,\"y\":0},\"panelIndex\":\"1c0ec894-1fd1-4a1a-8aa1-730e651ecca8\",\"version\":\"2.12.0\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":16,\"i\":\"d395e5d6-8064-4226-b4bd-7c21db0d6cf5\",\"w\":34,\"x\":14,\"y\":0},\"panelIndex\":\"d395e5d6-8064-4226-b4bd-7c21db0d6cf5\",\"version\":\"2.12.0\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":19,\"i\":\"4178095b-48cb-4ce8-b79c-eea288285dd5\",\"w\":48,\"x\":0,\"y\":16},\"panelIndex\":\"4178095b-48cb-4ce8-b79c-eea288285dd5\",\"version\":\"2.12.0\",\"panelRefName\":\"panel_2\"}]", "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", "version": 1, diff --git a/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json b/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json index 1ed2da6aa..5bc781396 100644 --- a/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json +++ b/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json @@ -375,7 +375,7 @@ "sort": [], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"event.provider:suricata AND event.kind:metric\",\"language\":\"lucene\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30m\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"query\":{\"query\":\"event.provider:suricata AND event.kind:metric\",\"language\":\"lucene\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30m\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ diff --git a/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json b/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json index 7312c147e..41fb3207a 100644 --- a/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json +++ b/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json @@ -12,7 +12,7 @@ "attributes": { "title": "Windows Events", "hits": 0, - "description": "", + "description": "Windows events and resource statistics sent via Fluent Bit", "panelsJSON": "[{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"346bb696-5fa2-4504-a1d8-5a6f51244c7b\",\"w\":18,\"x\":0,\"y\":0},\"panelIndex\":\"346bb696-5fa2-4504-a1d8-5a6f51244c7b\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":2,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_0\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"9c39d8b3-ad8c-4247-b97f-9736e469c988\",\"w\":30,\"x\":18,\"y\":0},\"panelIndex\":\"9c39d8b3-ad8c-4247-b97f-9736e469c988\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_1\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":36,\"i\":\"bccfb126-a864-4c11-a8c7-a9a1286c8f0f\",\"w\":30,\"x\":0,\"y\":15},\"panelIndex\":\"bccfb126-a864-4c11-a8c7-a9a1286c8f0f\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"bc116b54-f251-4e77-833c-c557b5d5c1d7\",\"w\":18,\"x\":30,\"y\":15},\"panelIndex\":\"bc116b54-f251-4e77-833c-c557b5d5c1d7\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}},\"table\":null},\"panelRefName\":\"panel_3\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":21,\"i\":\"5566d8f7-5c04-4f92-acf3-5af407b28b0d\",\"w\":18,\"x\":30,\"y\":30},\"panelIndex\":\"5566d8f7-5c04-4f92-acf3-5af407b28b0d\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_4\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":30,\"i\":\"49f38efc-1ab3-4e38-96e5-b0458c026491\",\"w\":48,\"x\":0,\"y\":66},\"panelIndex\":\"49f38efc-1ab3-4e38-96e5-b0458c026491\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":25,\"i\":\"0249a576-f162-4bf9-aa2d-bf85552665c7\",\"w\":48,\"x\":0,\"y\":96},\"panelIndex\":\"0249a576-f162-4bf9-aa2d-bf85552665c7\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"65476182-297e-4bbf-a7af-b0ff0584eeb0\",\"w\":24,\"x\":0,\"y\":51},\"panelIndex\":\"65476182-297e-4bbf-a7af-b0ff0584eeb0\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_7\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":24,\"y\":51,\"w\":24,\"h\":15,\"i\":\"0bf80b0d-91f3-47ca-b2a3-ea6844c9727a\"},\"panelIndex\":\"0bf80b0d-91f3-47ca-b2a3-ea6844c9727a\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_8\"}]", "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", "version": 1, diff --git a/dashboards/dashboards/beats/7a7e0a60-e8e8-11ec-b9d4-4569bb965430.json b/dashboards/dashboards/beats/7a7e0a60-e8e8-11ec-b9d4-4569bb965430.json index dcdf9e469..f815ffdc7 100644 --- a/dashboards/dashboards/beats/7a7e0a60-e8e8-11ec-b9d4-4569bb965430.json +++ b/dashboards/dashboards/beats/7a7e0a60-e8e8-11ec-b9d4-4569bb965430.json @@ -260,7 +260,7 @@ "sort": [], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"kuery\",\"query\":\"event.module:auditlog AND miscbeat.auditlog:*\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"kuery\",\"query\":\"event.module:auditlog AND miscbeat.auditlog:*\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ diff --git a/dashboards/dashboards/beats/903f42c0-f634-11ec-828d-2fb7a4a26e1f.json b/dashboards/dashboards/beats/903f42c0-f634-11ec-828d-2fb7a4a26e1f.json index d9ffc162c..34cfd2492 100644 --- a/dashboards/dashboards/beats/903f42c0-f634-11ec-828d-2fb7a4a26e1f.json +++ b/dashboards/dashboards/beats/903f42c0-f634-11ec-828d-2fb7a4a26e1f.json @@ -12,7 +12,7 @@ "attributes": { "title": "Malcolm Sensor File/Directory Integrity", "hits": 0, - "description": "", + "description": "AIDE file/directory integrity report for Malcolm sensors and aggregators", "panelsJSON": "[{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":18,\"h\":7,\"i\":\"1b520d1b-b30e-4216-8c83-3eff88564503\"},\"panelIndex\":\"1b520d1b-b30e-4216-8c83-3eff88564503\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_0\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":18,\"y\":0,\"w\":30,\"h\":14,\"i\":\"1a09a091-d27c-48df-a145-5a33ecc33ffb\"},\"panelIndex\":\"1a09a091-d27c-48df-a145-5a33ecc33ffb\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":7,\"w\":18,\"h\":28,\"i\":\"4c570390-7394-4129-b637-81e58e3fa066\"},\"panelIndex\":\"4c570390-7394-4129-b637-81e58e3fa066\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":18,\"y\":14,\"w\":30,\"h\":21,\"i\":\"3e3af3d3-f0e6-4472-b27a-a0cc71db516b\"},\"panelIndex\":\"3e3af3d3-f0e6-4472-b27a-a0cc71db516b\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":35,\"w\":48,\"h\":22,\"i\":\"93ccff9d-7037-4a11-9478-ee6f5341831e\"},\"panelIndex\":\"93ccff9d-7037-4a11-9478-ee6f5341831e\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"}]", "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", "version": 1, @@ -195,7 +195,7 @@ "sort": [], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"event.module:aide\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"event.module:aide\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ diff --git a/dashboards/dashboards/beats/Filebeat-nginx-logs.json b/dashboards/dashboards/beats/Filebeat-nginx-logs.json index 7e3bb7a02..879a0ae00 100644 --- a/dashboards/dashboards/beats/Filebeat-nginx-logs.json +++ b/dashboards/dashboards/beats/Filebeat-nginx-logs.json @@ -19,7 +19,7 @@ "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"version\":true,\"highlightAll\":true,\"filter\":[]}" + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"version\":true,\"highlightAll\":false,\"filter\":[]}" } }, "references": [ @@ -64,7 +64,7 @@ "description": "", "hits": 0, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"query\":{\"query\":\"event.module:nginx AND error.message:*\",\"language\":\"kuery\"},\"version\":true,\"highlight\":{\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"pre_tags\":[\"@kibana-highlighted-field@\"],\"require_field_match\":false,\"fragment_size\":2147483647},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"filter\":[],\"highlightAll\":false,\"query\":{\"query\":\"event.module:nginx AND error.message:*\",\"language\":\"kuery\"},\"version\":true,\"highlight\":{\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"pre_tags\":[\"@kibana-highlighted-field@\"],\"require_field_match\":false,\"fragment_size\":2147483647},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" }, "sort": [ [ @@ -104,7 +104,7 @@ "description": "", "hits": 0, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"query\":{\"query\":\"event.module:nginx AND url.original:*\",\"language\":\"kuery\"},\"version\":true,\"highlight\":{\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"pre_tags\":[\"@kibana-highlighted-field@\"],\"require_field_match\":false,\"fragment_size\":2147483647},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"filter\":[],\"highlightAll\":false,\"query\":{\"query\":\"event.module:nginx AND url.original:*\",\"language\":\"kuery\"},\"version\":true,\"highlight\":{\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"pre_tags\":[\"@kibana-highlighted-field@\"],\"require_field_match\":false,\"fragment_size\":2147483647},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" }, "sort": [ [ diff --git a/dashboards/dashboards/beats/Filebeat-nginx-overview.json b/dashboards/dashboards/beats/Filebeat-nginx-overview.json index ce8eed3fe..73e3fe9c5 100644 --- a/dashboards/dashboards/beats/Filebeat-nginx-overview.json +++ b/dashboards/dashboards/beats/Filebeat-nginx-overview.json @@ -13,7 +13,7 @@ "description": "nginx logs, including from Malcolm's own nginx instance", "hits": 0, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"version\":true,\"highlightAll\":true}" + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"version\":true,\"highlightAll\":false}" }, "optionsJSON": "{\"darkTheme\": false}", "panelsJSON": "[{\"panelIndex\":\"3\",\"panelRefName\":\"panel_0\",\"version\":\"7.3.0\",\"gridData\":{\"x\":36,\"y\":44,\"w\":12,\"h\":12,\"i\":\"3\"},\"embeddableConfig\":{}},{\"panelIndex\":\"4\",\"panelRefName\":\"panel_1\",\"version\":\"7.3.0\",\"gridData\":{\"x\":24,\"y\":44,\"w\":12,\"h\":12,\"i\":\"4\"},\"embeddableConfig\":{\"vis\":{\"legendOpen\":true}}},{\"panelIndex\":\"8\",\"panelRefName\":\"panel_2\",\"version\":\"7.3.0\",\"gridData\":{\"x\":0,\"y\":4,\"w\":48,\"h\":16,\"i\":\"8\"},\"embeddableConfig\":{\"mapZoom\":2,\"mapBounds\":{\"bottom_right\":{\"lat\":-7.362466865535738,\"lon\":245.39062500000003},\"top_left\":{\"lat\":77.07878389624943,\"lon\":-245.74218750000003}},\"mapCenter\":[50.51342652633956,-0.17578125],\"mapCollar\":{\"bottom_right\":{\"lat\":-49.583095,\"lon\":180},\"zoom\":2,\"top_left\":{\"lat\":90,\"lon\":-180}}}},{\"panelIndex\":\"13\",\"panelRefName\":\"panel_3\",\"version\":\"7.3.0\",\"gridData\":{\"x\":0,\"y\":20,\"w\":48,\"h\":12,\"i\":\"13\"},\"embeddableConfig\":{}},{\"panelIndex\":\"14\",\"panelRefName\":\"panel_4\",\"version\":\"7.3.0\",\"gridData\":{\"x\":24,\"y\":32,\"w\":24,\"h\":12,\"i\":\"14\"},\"embeddableConfig\":{}},{\"panelIndex\":\"15\",\"panelRefName\":\"panel_5\",\"version\":\"7.3.0\",\"gridData\":{\"x\":0,\"y\":32,\"w\":24,\"h\":12,\"i\":\"15\"},\"embeddableConfig\":{}},{\"panelIndex\":\"16\",\"panelRefName\":\"panel_6\",\"version\":\"7.3.0\",\"gridData\":{\"x\":0,\"y\":44,\"w\":24,\"h\":12,\"i\":\"16\"},\"embeddableConfig\":{}},{\"panelIndex\":\"17\",\"panelRefName\":\"panel_7\",\"version\":\"7.3.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":4,\"i\":\"17\"},\"embeddableConfig\":{}}]", @@ -289,7 +289,7 @@ "description": "", "hits": 0, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"query\":{\"query\":\"event.module:nginx\",\"language\":\"kuery\"},\"version\":true,\"highlight\":{\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"pre_tags\":[\"@kibana-highlighted-field@\"],\"require_field_match\":false,\"fragment_size\":2147483647},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"filter\":[],\"highlightAll\":false,\"query\":{\"query\":\"event.module:nginx\",\"language\":\"kuery\"},\"version\":true,\"highlight\":{\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"pre_tags\":[\"@kibana-highlighted-field@\"],\"require_field_match\":false,\"fragment_size\":2147483647},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" }, "sort": [ [ diff --git a/dashboards/dashboards/beats/Metricbeat-host-overview.json b/dashboards/dashboards/beats/Metricbeat-host-overview.json index a441acf0f..b0e401a3f 100644 --- a/dashboards/dashboards/beats/Metricbeat-host-overview.json +++ b/dashboards/dashboards/beats/Metricbeat-host-overview.json @@ -18,7 +18,7 @@ "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"host.name:\\\"hedgehogvm\\\"\"},\"version\":true,\"highlightAll\":true,\"filter\":[]}" + "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"host.name:\\\"hedgehogvm\\\"\"},\"version\":true,\"highlightAll\":false,\"filter\":[]}" } }, "references": [ diff --git a/dashboards/dashboards/beats/Metricbeat-system-overview.json b/dashboards/dashboards/beats/Metricbeat-system-overview.json index 407ccae86..eea8dd158 100644 --- a/dashboards/dashboards/beats/Metricbeat-system-overview.json +++ b/dashboards/dashboards/beats/Metricbeat-system-overview.json @@ -18,7 +18,7 @@ "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"version\":true,\"highlightAll\":true,\"filter\":[]}" + "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"version\":true,\"highlightAll\":false,\"filter\":[]}" } }, "references": [ diff --git a/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json b/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json index ca61e87d3..0ce72ec85 100644 --- a/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json +++ b/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json @@ -233,7 +233,7 @@ "sort": [], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"miscbeat.systemd:*\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"calendar_interval\":\"1w\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"query\":{\"query\":\"miscbeat.systemd:*\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"calendar_interval\":\"1w\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ From 1ed4758fcb5f5a144ff1d4aa82f37e085c421cc9 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 1 Mar 2024 15:47:32 -0700 Subject: [PATCH 71/79] fix error in dashboard --- .../beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json b/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json index 41fb3207a..d714f6d1f 100644 --- a/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json +++ b/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json @@ -246,7 +246,7 @@ "sort": [], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"event.module:(winlog OR winevtlog)\",\"language\":\"lucene\"},\"highlightAll\":false\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"query\":{\"query\":\"event.module:(winlog OR winevtlog)\",\"language\":\"lucene\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ @@ -285,7 +285,7 @@ "sort": [], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"event.module:winstat\",\"language\":\"kuery\"},\"highlightAll\":false\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"query\":{\"query\":\"event.module:winstat\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ From 76c8a7c22b57d07145f303dcdcc4384d1dd4f509 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 4 Mar 2024 07:27:02 -0700 Subject: [PATCH 72/79] tweak github image helper --- scripts/github_image_helper.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/github_image_helper.sh b/scripts/github_image_helper.sh index 931e8faa6..bf283c993 100755 --- a/scripts/github_image_helper.sh +++ b/scripts/github_image_helper.sh @@ -184,6 +184,7 @@ trap "_cleanup" EXIT FUNCTIONS=($(declare -F | awk '{print $NF}' | sort -f | egrep -v "^_")) # present the menu to our customer and get their selection +printf "%s\t%s\n" "0" "pull and extract everything" for i in "${!FUNCTIONS[@]}"; do ((IPLUS=i+1)) printf "%s\t%s\n" "$IPLUS" "${FUNCTIONS[$i]}" @@ -191,7 +192,13 @@ done echo -n "Operation:" [[ -n "${1-}" ]] && USER_FUNCTION_IDX="$1" || read USER_FUNCTION_IDX -if (( $USER_FUNCTION_IDX > 0 )) && (( $USER_FUNCTION_IDX <= "${#FUNCTIONS[@]}" )); then +if (( $USER_FUNCTION_IDX == 0 )); then + PullAndTagGithubWorkflowISOImages + ExtractAndLoadImagesFromGithubWorkflowBuildISO + ExtractISOsFromGithubWorkflowBuilds + PullAndTagGithubWorkflowImages + +elif (( $USER_FUNCTION_IDX > 0 )) && (( $USER_FUNCTION_IDX <= "${#FUNCTIONS[@]}" )); then # execute one function, à la carte USER_FUNCTION="${FUNCTIONS[((USER_FUNCTION_IDX-1))]}" echo $USER_FUNCTION From 23eff1625d8629ee755162cfc6866ac580706d38 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 4 Mar 2024 10:29:17 -0700 Subject: [PATCH 73/79] github_image_helper script updates --- scripts/github_image_helper.sh | 35 +++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/scripts/github_image_helper.sh b/scripts/github_image_helper.sh index bf283c993..a3f496eee 100755 --- a/scripts/github_image_helper.sh +++ b/scripts/github_image_helper.sh @@ -1,5 +1,21 @@ #!/usr/bin/env bash +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. + +if [ -z "$BASH_VERSION" ]; then + echo "Wrong interpreter, please run \"$0\" with bash" + exit 1 +fi + +[[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath +[[ "$(uname -s)" = 'Darwin' ]] && DIRNAME=gdirname || DIRNAME=dirname +[[ "$(uname -s)" = 'Darwin' ]] && GREP=ggrep || GREP=grep +if ! (type "$REALPATH" && type "$DIRNAME" && type "$GREP" && type git) > /dev/null; then + echo "$(basename "${BASH_SOURCE[0]}") requires $REALPATH and $DIRNAME and $GREP and git" + exit 1 +fi +SCRIPT_PATH="$($DIRNAME $($REALPATH -e "${BASH_SOURCE[0]}"))" + set -uo pipefail shopt -s nocasematch ENCODING="utf-8" @@ -32,32 +48,42 @@ function _cols() { # get the current git working copy's branch (e.g., main) function _gitbranch() { + pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 git rev-parse --abbrev-ref HEAD + popd >/dev/null 2>&1 } # get the current git working copy's remote name (e.g., origin) function _gitremote() { - git branch -vv | grep "^\*" | cut -d "[" -f2 | cut -d "]" -f1 | cut -d "/" -f1 + pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 + git branch -vv | $GREP "^\*" | cut -d "[" -f2 | cut -d "]" -f1 | cut -d "/" -f1 + popd >/dev/null 2>&1 } # get the current git working copy's top-level directory function _gittoplevel() { + pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 git rev-parse --show-toplevel + popd >/dev/null 2>&1 } # get the current git working copy's remote "owner" (github user or organization, e.g., johndoe) function _gitowner() { + pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 git remote get-url "$(_gitremote)" | sed 's@.*github\.com/@@' | cut -d'/' -f1 + popd >/dev/null 2>&1 } # get the current git working copy's remote repository name (e.g., malcolm) function _gitreponame() { + pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 git remote get-url "$(_gitremote)" | sed 's@.*github\.com/@@' | cut -d'/' -f2 + popd >/dev/null 2>&1 } # get the current git working copy's Malcolm version (grepped from docker-compose.yml, e.g., 5.0.3) function _malcolmversion() { - grep -P "^\s+image:.*/malcolm" "$(_gittoplevel)"/docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1 + $GREP -P "^\s+image:.*/malcolm" "$(_gittoplevel)"/docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1 } ################################################################################ @@ -88,7 +114,7 @@ function PullAndTagGithubWorkflowImages() { VERSION="$(_malcolmversion)" OWNER="$(_gitowner)" echo "Pulling images from ghcr.io/$OWNER ($BRANCH) and tagging as $VERSION ..." - for IMG in $(grep image: "$(_gittoplevel)"/docker-compose.yml | _cols 2 | cut -d: -f1 | sort -u | sed "s/.*\/\(malcolm\)/\1/"); do + for IMG in $($GREP image: "$(_gittoplevel)"/docker-compose.yml | _cols 2 | cut -d: -f1 | sort -u | sed "s/.*\/\(malcolm\)/\1/"); do _PullAndTagGithubWorkflowBuild "$IMG" done echo "done" @@ -180,6 +206,9 @@ function GithubTriggerPackagesBuild () { trap "_cleanup" EXIT +# force-navigate to Malcolm base directory (parent of scripts/ directory) +pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 + # get a list of all the "public" functions (not starting with _) FUNCTIONS=($(declare -F | awk '{print $NF}' | sort -f | egrep -v "^_")) From 8bd8e44065fa0ed4830408a495e5ae3caa4027b4 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 4 Mar 2024 11:12:08 -0700 Subject: [PATCH 74/79] fix renaming some json fields --- logstash/pipelines/zeek/11_zeek_parse.conf | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index 7f34c7f74..cd68487ee 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -5815,7 +5815,17 @@ filter { # broker.log # https://docs.zeek.org/en/master/scripts/base/frameworks/broker/log.zeek.html - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_broker_fields" + rename => { "[zeek_cols][ty]" => "[zeek_cols][event_type]" } + rename => { "[zeek_cols][ev]" => "[zeek_cols][event_action]" } + rename => { "[zeek_cols][peer.address]" => "[zeek_cols][peer_ip]" } + rename => { "[zeek_cols][peer.bound_port]" => "[zeek_cols][peer_port]" } + rename => { "[zeek_cols][message]" => "[zeek_cols][peer_message]" } + } + + } else { dissect { id => "dissect_zeek_diagnostic_broker" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -5877,7 +5887,13 @@ filter { # Logging for establishing and controlling a cluster of Zeek instances # https://docs.zeek.org/en/master/scripts/base/frameworks/cluster/main.zeek.html#type-Cluster::Info - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_cluster_fields" + rename => { "[zeek_cols][message]" => "[zeek_cols][node_message]" } + } + + } else { dissect { id => "dissect_zeek_diagnostic_cluster" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP From ba26fa6d306a90f93b0f331099b443e809f42ac6 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 4 Mar 2024 11:16:43 -0700 Subject: [PATCH 75/79] fix renaming some json fields --- .../composable/component/zeek_diagnostic.json | 8 +++----- logstash/pipelines/zeek/11_zeek_parse.conf | 10 ++++++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/dashboards/templates/composable/component/zeek_diagnostic.json b/dashboards/templates/composable/component/zeek_diagnostic.json index eaa65e284..9aac62141 100644 --- a/dashboards/templates/composable/component/zeek_diagnostic.json +++ b/dashboards/templates/composable/component/zeek_diagnostic.json @@ -26,11 +26,9 @@ "packet_filter.success": { "type": "keyword" }, "packet_filter.failure_reason": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, "print.vals": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, - "reporter.node": { "type": "keyword" }, - "reporter.filter": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, - "reporter.init": { "type": "keyword" }, - "reporter.success": { "type": "keyword" }, - "reporter.failure_reason": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "reporter.level": { "type": "keyword" }, + "reporter.message": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "reporter.location": { "type": "keyword" }, "stats.peer": { "type": "keyword" }, "stats.mem": { "type": "long" }, "stats.pkts_proc": { "type": "long" }, diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index cd68487ee..c539aca21 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -6017,7 +6017,13 @@ filter { # reporter.log # https://docs.zeek.org/en/master/scripts/base/frameworks/reporter/main.zeek.html#type-Reporter::Info - if ("_jsonparsesuccess" not in [tags]) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_reporter_fields" + rename => { "[zeek_cols][message]" => "[zeek_cols][msg]" } + } + + } else { dissect { id => "dissect_zeek_diagnostic_reporter" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -6033,7 +6039,7 @@ filter { } ruby { id => "ruby_zip_zeek_diagnostic_reporter" - init => "@zeek_diagnostic_reporter_field_names = [ 'ts', 'node', 'filter', 'init', 'success', 'failure_reason' ]" + init => "@zeek_diagnostic_reporter_field_names = [ 'ts', 'level', 'msg', 'location' ]" code => "event.set('[zeek_cols]', @zeek_diagnostic_reporter_field_names.zip(event.get('[message]')).to_h)" } } From 6fa20be9980858bb137c1aeb0bc6221459a2ead1 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 4 Mar 2024 13:13:53 -0700 Subject: [PATCH 76/79] clean up some leftover files --- hedgehog-raspi/sensor_install.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hedgehog-raspi/sensor_install.sh b/hedgehog-raspi/sensor_install.sh index d15225bcd..5e607bdbf 100644 --- a/hedgehog-raspi/sensor_install.sh +++ b/hedgehog-raspi/sensor_install.sh @@ -270,8 +270,12 @@ clean_up() { /opt/hedgehog_install_artifacts \ /opt/hooks \ /opt/patches \ - /root/.bash_history \ - /root/.wget-hsts \ + /root/.bash_history \ + /root/.wget-hsts \ + /root/.cache \ + /root/.local/share/gem \ + /root/.npm \ + "${DEBS_DIR}" \ /tmp/* find /var/log/ -type f -print0 2>/dev/null | \ xargs -0 -r -I XXX bash -c "file 'XXX' | grep -q text && > 'XXX'" From 4bd86766d5eb818318bf8105e6afcfafa87c36d1 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 4 Mar 2024 14:09:36 -0700 Subject: [PATCH 77/79] store a cumulative pkts_dropped for suircata --- .../4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json | 70 +++++++++---------- .../composable/component/suricata_stats.json | 1 + logstash/pipelines/beats/11_beats_logs.conf | 11 +++ 3 files changed, 47 insertions(+), 35 deletions(-) diff --git a/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json b/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json index 5bc781396..7a5bb0416 100644 --- a/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json +++ b/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json @@ -7,8 +7,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T19:19:33.901Z", - "version": "WzEwOTksMV0=", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg4NCwxXQ==", "attributes": { "title": "Packet Capture Statistics", "hits": 0, @@ -108,8 +108,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T19:12:21.755Z", - "version": "WzEwOTUsMV0=", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg4NSwxXQ==", "attributes": { "title": "Last Capture Metric Timestamp by Host", "visState": "{\"title\":\"Last Capture Metric Timestamp by Host\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"top_hits\",\"params\":{\"field\":\"@timestamp\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Last Metric Timestamp\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"host.name\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Capture Host\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.provider\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":5,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Other\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", @@ -137,11 +137,11 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T18:50:17.404Z", - "version": "Wzk5NywxXQ==", + "updated_at": "2024-03-04T21:07:41.024Z", + "version": "Wzk5OSwxXQ==", "attributes": { "title": "Zeek and Suricata Capture Measurements ", - "visState": "{\"title\":\"Zeek and Suricata Capture Measurements \",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"32d1fca0-d7e1-11ee-ad81-217e54128a4b\",\"color\":\"rgba(33,150,243,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"32d1fca1-d7e1-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_link\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Zeek: packets seen\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"32d1fca1-d7e1-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"02bbf6a0-d7fb-11ee-a5f1-9ff9da698a18\",\"color\":\"rgba(84,179,153,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"unit\":\"\",\"id\":\"02bbf6a1-d7fb-11ee-a5f1-9ff9da698a18\",\"type\":\"positive_rate\",\"field\":\"suricata.stats.capture.kernel_packets\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Suricata: packets seen\",\"type\":\"timeseries\"},{\"id\":\"e4143600-d7e0-11ee-ad81-217e54128a4b\",\"color\":\"rgba(229,115,115,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_dropped\"},{\"id\":\"f6df2790-d7e0-11ee-ad81-217e54128a4b\",\"type\":\"math\",\"variables\":[{\"id\":\"f8ee0a60-d7e0-11ee-ad81-217e54128a4b\",\"name\":\"packets\",\"field\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\"}],\"script\":\"params.packets*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Zeek: packets dropped\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"20b9a420-d7df-11ee-ad81-217e54128a4b\",\"color\":\"rgba(211,96,134,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.capture_loss.gaps\"},{\"id\":\"9a3afce0-d7df-11ee-ad81-217e54128a4b\",\"type\":\"math\",\"variables\":[{\"id\":\"9dece150-d7df-11ee-ad81-217e54128a4b\",\"name\":\"gaps\",\"field\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\"}],\"script\":\"params.gaps*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Zeek: ACKS missed\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"cad40600-d7fb-11ee-a5f1-9ff9da698a18\",\"color\":\"rgba(255,171,145,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"unit\":\"\",\"id\":\"cad40601-d7fb-11ee-a5f1-9ff9da698a18\",\"type\":\"positive_rate\",\"field\":\"suricata.stats.capture.kernel_drops\"},{\"id\":\"f5352cd0-d7fb-11ee-a5f1-9ff9da698a18\",\"type\":\"math\",\"variables\":[{\"id\":\"f79def70-d7fb-11ee-a5f1-9ff9da698a18\",\"name\":\"packets\",\"field\":\"cad40601-d7fb-11ee-a5f1-9ff9da698a18\"}],\"script\":\"params.packets*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Suricata: packets dropped\",\"type\":\"timeseries\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"filter\":{\"query\":\"(event.provider:zeek OR event.provider:suricata) AND event.kind:metric\",\"language\":\"kuery\"},\"legend_position\":\"right\",\"background_color\":null}}", + "visState": "{\"title\":\"Zeek and Suricata Capture Measurements \",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"32d1fca0-d7e1-11ee-ad81-217e54128a4b\",\"color\":\"rgba(33,150,243,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"32d1fca1-d7e1-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_link\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Zeek: packets seen\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"32d1fca1-d7e1-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"02bbf6a0-d7fb-11ee-a5f1-9ff9da698a18\",\"color\":\"rgba(84,179,153,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"unit\":\"\",\"id\":\"02bbf6a1-d7fb-11ee-a5f1-9ff9da698a18\",\"type\":\"positive_rate\",\"field\":\"suricata.stats.capture.kernel_packets\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Suricata: packets seen\",\"type\":\"timeseries\"},{\"id\":\"e4143600-d7e0-11ee-ad81-217e54128a4b\",\"color\":\"rgba(229,115,115,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_dropped\"},{\"id\":\"f6df2790-d7e0-11ee-ad81-217e54128a4b\",\"type\":\"math\",\"variables\":[{\"id\":\"f8ee0a60-d7e0-11ee-ad81-217e54128a4b\",\"name\":\"packets\",\"field\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\"}],\"script\":\"params.packets*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Zeek: packets dropped\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"20b9a420-d7df-11ee-ad81-217e54128a4b\",\"color\":\"rgba(211,96,134,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.capture_loss.gaps\"},{\"id\":\"9a3afce0-d7df-11ee-ad81-217e54128a4b\",\"type\":\"math\",\"variables\":[{\"id\":\"9dece150-d7df-11ee-ad81-217e54128a4b\",\"name\":\"gaps\",\"field\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\"}],\"script\":\"params.gaps*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Zeek: ACKS missed\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"cad40600-d7fb-11ee-a5f1-9ff9da698a18\",\"color\":\"rgba(255,171,145,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"unit\":\"\",\"id\":\"cad40601-d7fb-11ee-a5f1-9ff9da698a18\",\"type\":\"positive_rate\",\"field\":\"suricata.stats.pkts_dropped\"},{\"id\":\"f5352cd0-d7fb-11ee-a5f1-9ff9da698a18\",\"type\":\"math\",\"variables\":[{\"id\":\"f79def70-d7fb-11ee-a5f1-9ff9da698a18\",\"name\":\"packets\",\"field\":\"cad40601-d7fb-11ee-a5f1-9ff9da698a18\"}],\"script\":\"params.packets*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Suricata: packets dropped\",\"type\":\"timeseries\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"filter\":{\"query\":\"(event.provider:zeek OR event.provider:suricata) AND event.kind:metric\",\"language\":\"kuery\"},\"legend_position\":\"right\",\"background_color\":null}}", "uiStateJSON": "{}", "description": "Positive values on the y-axis represent observed packets while negative values represent missing dropped packets and missing ACKs.\n\nThis data is logged by Zeek in stats.log (https://docs.zeek.org/en/master/scripts/policy/misc/stats.zeek.html#type-Stats::Info) and capture_loss.log (https://docs.zeek.org/en/master/scripts/policy/misc/capture-loss.zeek.html#type-CaptureLoss::Info), and by Suricata (https://docs.suricata.io/en/suricata-6.0.0/performance/statistics.html).", "version": 1, @@ -160,8 +160,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T16:02:17.775Z", - "version": "Wzk2OSwxXQ==", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg4NywxXQ==", "attributes": { "title": "Zeek Stats - Packets and Bytes", "visState": "{\"title\":\"Zeek Stats - Packets and Bytes\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_link\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Total Packets Seen\"},{\"id\":\"bd4560e0-d7e4-11ee-ad81-217e54128a4b\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"bd4560e1-d7e4-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.bytes_recv\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Total Bytes Seen\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:zeek AND event.dataset:stats\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", @@ -183,8 +183,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T16:12:29.036Z", - "version": "Wzk3NCwxXQ==", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg4OCwxXQ==", "attributes": { "title": "Zeek Stats - Capture Loss", "visState": "{\"title\":\"Zeek Stats - Capture Loss\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_dropped\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Packets Dropped\"},{\"id\":\"bd4560e0-d7e4-11ee-ad81-217e54128a4b\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"bd4560e1-d7e4-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.capture_loss.gaps\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"ACKs Missed\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:zeek AND event.dataset:(stats OR capture_loss)\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", @@ -206,8 +206,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T15:01:44.258Z", - "version": "Wzg4MywxXQ==", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg4OSwxXQ==", "attributes": { "title": "Packet Capture - Zeek capture_loss.log", "description": "", @@ -243,8 +243,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T18:56:30.009Z", - "version": "WzEwMDksMV0=", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5MCwxXQ==", "attributes": { "title": "Suricata Stats - Packets and Bytes", "visState": "{\"title\":\"Suricata Stats - Packets and Bytes\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"unit\":\"\",\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"top_hit\",\"field\":\"suricata.stats.capture.kernel_packets\",\"order_by\":\"@timestamp\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Total Packets Seen\"},{\"id\":\"bd4560e0-d7e4-11ee-ad81-217e54128a4b\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"bd4560e1-d7e4-11ee-ad81-217e54128a4b\",\"type\":\"top_hit\",\"field\":\"suricata.stats.decoder.bytes\",\"order_by\":\"@timestamp\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Total Bytes Seen\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:suricata AND event.kind:metric\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", @@ -266,11 +266,11 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T19:02:50.252Z", - "version": "WzEwNzksMV0=", + "updated_at": "2024-03-04T21:07:12.009Z", + "version": "Wzk5NiwxXQ==", "attributes": { "title": "Suricata Stats - Capture Loss", - "visState": "{\"title\":\"Suricata Stats - Capture Loss\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"top_hit\",\"field\":\"suricata.stats.capture.kernel_drops\",\"order_by\":\"@timestamp\"},{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"ffb34320-d7fd-11ee-a5f1-9ff9da698a18\",\"type\":\"top_hit\",\"field\":\"suricata.stats.tcp.segment_memcap_drop\",\"order_by\":\"@timestamp\"},{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"0e962510-d7fe-11ee-a5f1-9ff9da698a18\",\"type\":\"top_hit\",\"field\":\"suricata.stats.tcp.ssn_memcap_drop\",\"order_by\":\"@timestamp\"},{\"id\":\"1d60c410-d7fe-11ee-a5f1-9ff9da698a18\",\"type\":\"math\",\"variables\":[{\"id\":\"21851960-d7fe-11ee-a5f1-9ff9da698a18\",\"name\":\"kernel\",\"field\":\"61ca57f2-469d-11e7-af02-69e470af7417\"},{\"id\":\"25a01fe0-d7fe-11ee-a5f1-9ff9da698a18\",\"name\":\"segment\",\"field\":\"ffb34320-d7fd-11ee-a5f1-9ff9da698a18\"},{\"id\":\"2b81b590-d7fe-11ee-a5f1-9ff9da698a18\",\"name\":\"session\",\"field\":\"0e962510-d7fe-11ee-a5f1-9ff9da698a18\"}],\"script\":\"params.kernel+params.segment+params.session\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Packets Dropped\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:suricata AND event.kind:metric\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", + "visState": "{\"title\":\"Suricata Stats - Capture Loss\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"top_hit\",\"field\":\"suricata.stats.pkts_dropped\",\"order_by\":\"@timestamp\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Packets Dropped\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:suricata AND event.kind:metric\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", "uiStateJSON": "{}", "description": "", "version": 1, @@ -289,8 +289,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T15:01:49.331Z", - "version": "WzkxNywxXQ==", + "updated_at": "2024-03-04T21:05:59.817Z", + "version": "Wzk0MCwxXQ==", "attributes": { "title": "Network Traffic (Packets)", "visState": "{\"title\":\"Network Traffic (Packets)\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"axis_scale\":\"normal\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"filter\":{\"language\":\"lucene\",\"query\":\"\"},\"id\":\"da1046f0-faa0-11e6-86b1-cd7735ff7e23\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"auto\",\"isModelInvalid\":false,\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"rgba(0,133,255,1)\",\"fill\":\"1\",\"formatter\":\"'0a'\",\"id\":\"49931900-ebf3-11ec-a401-f5db2d59e6af\",\"label\":\"Inbound\",\"line_width\":1,\"metrics\":[{\"unit\":\"1s\",\"id\":\"49931901-ebf3-11ec-a401-f5db2d59e6af\",\"type\":\"positive_rate\",\"field\":\"miscbeat.network.packets.rx\"}],\"point_size\":1,\"separate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"type\":\"timeseries\",\"terms_field\":\"miscbeat.network.interface\",\"terms_size\":\"3\",\"terms_order_by\":\"_count\",\"value_template\":\"{{value}}/s\",\"split_color_mode\":\"gradient\"},{\"id\":\"75fba890-ebf3-11ec-a401-f5db2d59e6af\",\"color\":\"rgba(13,212,26,1)\",\"split_mode\":\"terms\",\"metrics\":[{\"unit\":\"1s\",\"id\":\"75fba891-ebf3-11ec-a401-f5db2d59e6af\",\"type\":\"positive_rate\",\"field\":\"miscbeat.network.packets.tx\"},{\"id\":\"96daba60-ebf3-11ec-a401-f5db2d59e6af\",\"type\":\"math\",\"variables\":[{\"id\":\"98e138c0-ebf3-11ec-a401-f5db2d59e6af\",\"name\":\"rate\",\"field\":\"75fba891-ebf3-11ec-a401-f5db2d59e6af\"}],\"script\":\"params.rate*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"'0a'\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":\"1\",\"stacked\":\"none\",\"label\":\"Outbound\",\"type\":\"timeseries\",\"terms_size\":\"3\",\"terms_field\":\"miscbeat.network.interface\",\"terms_order_by\":\"_count\",\"split_color_mode\":\"gradient\",\"value_template\":\"{{value}}/s\"}],\"show_grid\":1,\"show_legend\":1,\"time_field\":\"@timestamp\",\"tooltip_mode\":\"show_all\",\"type\":\"timeseries\"}}", @@ -312,8 +312,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T15:01:44.258Z", - "version": "Wzg4NCwxXQ==", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5MywxXQ==", "attributes": { "title": "Packet Capture - Zeek stats.log", "description": "", @@ -354,8 +354,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T18:37:00.084Z", - "version": "Wzk5NCwxXQ==", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5NCwxXQ==", "attributes": { "title": "Packet Capture - Suricata Stats", "description": "", @@ -363,7 +363,7 @@ "columns": [ "host.name", "suricata.stats.capture.kernel_packets", - "suricata.stats.capture.kernel_drops", + "suricata.stats.pkts_dropped", "suricata.stats.capture.errors", "suricata.stats.decoder.bytes", "suricata.stats.decoder.ethernet", @@ -395,8 +395,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T19:07:58.499Z", - "version": "WzEwOTEsMV0=", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5NSwxXQ==", "attributes": { "title": "Zeek Analyzer Messages", "visState": "{\"title\":\"Zeek Analyzer Messages\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.cause\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Cause\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.analyzer_kind\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Class\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.analyzer_name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Analyzer\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", @@ -425,8 +425,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T15:01:44.258Z", - "version": "Wzg4MiwxXQ==", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5NiwxXQ==", "attributes": { "title": "Packet Capture - Zeek analyzer.log", "description": "", @@ -464,8 +464,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T15:52:59.726Z", - "version": "Wzk2OCwxXQ==", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5NywxXQ==", "attributes": { "title": "Zeek - Reporter Categories", "visState": "{\"title\":\"Zeek - Reporter Categories\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.reporter.level.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100}}}", @@ -494,8 +494,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T15:01:44.258Z", - "version": "Wzg4NSwxXQ==", + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5OCwxXQ==", "attributes": { "title": "Packet Capture - Zeek reporter.log", "description": "", @@ -529,8 +529,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-03-01T15:01:49.331Z", - "version": "WzkxOCwxXQ==", + "updated_at": "2024-03-04T21:05:59.817Z", + "version": "Wzk0MSwxXQ==", "attributes": { "title": "Network Traffic (Bytes)", "visState": "{\"title\":\"Network Traffic (Bytes)\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"axis_scale\":\"normal\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"filter\":{\"language\":\"lucene\",\"query\":\"\"},\"id\":\"da1046f0-faa0-11e6-86b1-cd7735ff7e23\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"auto\",\"isModelInvalid\":false,\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"rgba(0,133,255,1)\",\"fill\":\"1\",\"formatter\":\"bytes\",\"id\":\"6d8b8ab0-ebf1-11ec-a401-f5db2d59e6af\",\"line_width\":1,\"metrics\":[{\"unit\":\"1s\",\"id\":\"6d8b8ab1-ebf1-11ec-a401-f5db2d59e6af\",\"type\":\"positive_rate\",\"field\":\"miscbeat.network.bytes.rx\"}],\"point_size\":1,\"separate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"label\":\"Inbound\",\"type\":\"timeseries\",\"terms_field\":\"miscbeat.network.interface\",\"terms_size\":\"3\",\"terms_order_by\":\"_key\",\"value_template\":\"{{value}}/s\",\"split_color_mode\":\"gradient\"},{\"id\":\"b5977de0-ebf2-11ec-a401-f5db2d59e6af\",\"color\":\"rgba(13,212,26,1)\",\"split_mode\":\"terms\",\"metrics\":[{\"unit\":\"1s\",\"id\":\"b5977de1-ebf2-11ec-a401-f5db2d59e6af\",\"type\":\"positive_rate\",\"field\":\"miscbeat.network.bytes.tx\"},{\"id\":\"cdfb1540-ebf2-11ec-a401-f5db2d59e6af\",\"type\":\"math\",\"variables\":[{\"id\":\"d1b9caf0-ebf2-11ec-a401-f5db2d59e6af\",\"name\":\"rate\",\"field\":\"b5977de1-ebf2-11ec-a401-f5db2d59e6af\"}],\"script\":\"params.rate*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":\"1\",\"stacked\":\"none\",\"label\":\"Outbound\",\"split_color_mode\":\"gradient\",\"type\":\"timeseries\",\"terms_size\":\"3\",\"terms_order_by\":\"_key\",\"terms_field\":\"miscbeat.network.interface\",\"value_template\":\"{{value}}/s\"}],\"show_grid\":1,\"show_legend\":1,\"time_field\":\"@timestamp\",\"tooltip_mode\":\"show_all\",\"type\":\"timeseries\"}}", diff --git a/dashboards/templates/composable/component/suricata_stats.json b/dashboards/templates/composable/component/suricata_stats.json index 03fb77b2c..3aeb82a91 100644 --- a/dashboards/templates/composable/component/suricata_stats.json +++ b/dashboards/templates/composable/component/suricata_stats.json @@ -169,6 +169,7 @@ "ftp.memuse": { "type": "long" }, "http.memcap": { "type": "long" }, "http.memuse": { "type": "long" }, + "pkts_dropped": { "type": "long" }, "tcp.insert_data_normal_fail": { "type": "long" }, "tcp.insert_data_overlap_fail": { "type": "long" }, "tcp.insert_list_fail": { "type": "long" }, diff --git a/logstash/pipelines/beats/11_beats_logs.conf b/logstash/pipelines/beats/11_beats_logs.conf index 979529161..2d9febd02 100644 --- a/logstash/pipelines/beats/11_beats_logs.conf +++ b/logstash/pipelines/beats/11_beats_logs.conf @@ -837,6 +837,17 @@ filter { # Suricata statistics # https://docs.suricata.io/en/suricata-6.0.2/configuration/suricata-yaml.html#stats + # sum packet drops into suricata.stats.pkts_dropped + ruby { + id => "ruby_miscbeat_suricata_stats_pkts_dropped" + code => " + pkts_dropped = event.get('[suricata][stats][capture][kernel_drops]').to_i + + event.get('[suricata][stats][tcp][segment_memcap_drop]').to_i + + event.get('[suricata][stats][tcp][ssn_memcap_drop]').to_i + event.set('[suricata][stats][pkts_dropped]', pkts_dropped) unless (pkts_dropped == 0) + " + } + if ([suricata][stats]) { # remove zero values from suricata stats ruby { From 80b6ec4e7c4ba7309c1ed8dd80249b7873e5adc7 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 4 Mar 2024 15:55:59 -0700 Subject: [PATCH 78/79] minor tweak to reporter dashboard --- .../beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json | 6 +++--- .../templates/composable/component/zeek_diagnostic.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json b/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json index 7a5bb0416..e17788646 100644 --- a/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json +++ b/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json @@ -13,7 +13,7 @@ "title": "Packet Capture Statistics", "hits": 0, "description": "Statistics and diagnostics for packet capture from Zeek and Suricata", - "panelsJSON": "[{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":13,\"h\":15,\"i\":\"0c179e97-9bcf-4f72-b717-b7a93667c1a0\"},\"panelIndex\":\"0c179e97-9bcf-4f72-b717-b7a93667c1a0\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":0,\"w\":35,\"h\":15,\"i\":\"b483d809-a528-4280-b79e-aa7ada17d275\"},\"panelIndex\":\"b483d809-a528-4280-b79e-aa7ada17d275\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_1\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":15,\"w\":13,\"h\":10,\"i\":\"e10dc0a6-f197-4cbc-a1ad-e67194f95a63\"},\"panelIndex\":\"e10dc0a6-f197-4cbc-a1ad-e67194f95a63\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_2\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":15,\"w\":13,\"h\":10,\"i\":\"01b20859-4d95-47e0-a536-6b1e9932c35b\"},\"panelIndex\":\"01b20859-4d95-47e0-a536-6b1e9932c35b\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_3\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":26,\"y\":15,\"w\":22,\"h\":20,\"i\":\"8e013ce7-3205-4d06-a805-6285826c1c5d\"},\"panelIndex\":\"8e013ce7-3205-4d06-a805-6285826c1c5d\",\"embeddableConfig\":{\"columns\":[\"@timestamp\",\"host.name\",\"zeek.capture_loss.peer\",\"zeek.capture_loss.acks\",\"zeek.capture_loss.gaps\",\"zeek.capture_loss.percent_lost\"]},\"panelRefName\":\"panel_4\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":25,\"w\":13,\"h\":10,\"i\":\"147b45ae-804b-4d9e-a9a9-806772ad3b35\"},\"panelIndex\":\"147b45ae-804b-4d9e-a9a9-806772ad3b35\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_5\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":25,\"w\":13,\"h\":10,\"i\":\"687597e3-4848-4629-8b85-45c0773efb79\"},\"panelIndex\":\"687597e3-4848-4629-8b85-45c0773efb79\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_6\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":35,\"w\":24,\"h\":15,\"i\":\"0174654c-2010-463a-b49e-fa5759b61b9c\"},\"panelIndex\":\"0174654c-2010-463a-b49e-fa5759b61b9c\",\"embeddableConfig\":{},\"panelRefName\":\"panel_7\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":50,\"w\":48,\"h\":21,\"i\":\"36e03a4a-e017-42b8-82cf-205d26b2ed6b\"},\"panelIndex\":\"36e03a4a-e017-42b8-82cf-205d26b2ed6b\",\"embeddableConfig\":{},\"panelRefName\":\"panel_8\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":71,\"w\":48,\"h\":21,\"i\":\"e1c0f1e0-de36-4527-bafa-a297fe9452a2\"},\"panelIndex\":\"e1c0f1e0-de36-4527-bafa-a297fe9452a2\",\"embeddableConfig\":{},\"panelRefName\":\"panel_9\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":92,\"w\":13,\"h\":20,\"i\":\"74a841b8-2ffc-4f6d-8b5a-ca7960eb6b10\"},\"panelIndex\":\"74a841b8-2ffc-4f6d-8b5a-ca7960eb6b10\",\"embeddableConfig\":{},\"panelRefName\":\"panel_10\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":92,\"w\":35,\"h\":20,\"i\":\"f15e46fe-040f-4602-ad13-01aab36b372a\"},\"panelIndex\":\"f15e46fe-040f-4602-ad13-01aab36b372a\",\"embeddableConfig\":{},\"panelRefName\":\"panel_11\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":112,\"w\":16,\"h\":17,\"i\":\"bfdc6d50-66f1-4f9a-9ea5-cd30bc01099d\"},\"panelIndex\":\"bfdc6d50-66f1-4f9a-9ea5-cd30bc01099d\",\"embeddableConfig\":{},\"panelRefName\":\"panel_12\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":16,\"y\":112,\"w\":32,\"h\":17,\"i\":\"efbd7f15-5af7-4e39-9889-c1c944a40dc2\"},\"panelIndex\":\"efbd7f15-5af7-4e39-9889-c1c944a40dc2\",\"embeddableConfig\":{\"columns\":[\"@timestamp\",\"host.name\",\"zeek.reporter.level\",\"zeek.reporter.message\",\"zeek.reporter.location\"]},\"panelRefName\":\"panel_13\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":24,\"y\":35,\"w\":24,\"h\":15,\"i\":\"2ecc4ac3-d694-46ab-a6b1-9c86e5e9d394\"},\"panelIndex\":\"2ecc4ac3-d694-46ab-a6b1-9c86e5e9d394\",\"embeddableConfig\":{},\"panelRefName\":\"panel_14\"}]", + "panelsJSON": "[{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":13,\"h\":15,\"i\":\"0c179e97-9bcf-4f72-b717-b7a93667c1a0\"},\"panelIndex\":\"0c179e97-9bcf-4f72-b717-b7a93667c1a0\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":0,\"w\":35,\"h\":15,\"i\":\"b483d809-a528-4280-b79e-aa7ada17d275\"},\"panelIndex\":\"b483d809-a528-4280-b79e-aa7ada17d275\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_1\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":15,\"w\":13,\"h\":10,\"i\":\"e10dc0a6-f197-4cbc-a1ad-e67194f95a63\"},\"panelIndex\":\"e10dc0a6-f197-4cbc-a1ad-e67194f95a63\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_2\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":15,\"w\":13,\"h\":10,\"i\":\"01b20859-4d95-47e0-a536-6b1e9932c35b\"},\"panelIndex\":\"01b20859-4d95-47e0-a536-6b1e9932c35b\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_3\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":26,\"y\":15,\"w\":22,\"h\":20,\"i\":\"8e013ce7-3205-4d06-a805-6285826c1c5d\"},\"panelIndex\":\"8e013ce7-3205-4d06-a805-6285826c1c5d\",\"embeddableConfig\":{\"columns\":[\"@timestamp\",\"host.name\",\"zeek.capture_loss.peer\",\"zeek.capture_loss.acks\",\"zeek.capture_loss.gaps\",\"zeek.capture_loss.percent_lost\"]},\"panelRefName\":\"panel_4\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":25,\"w\":13,\"h\":10,\"i\":\"147b45ae-804b-4d9e-a9a9-806772ad3b35\"},\"panelIndex\":\"147b45ae-804b-4d9e-a9a9-806772ad3b35\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_5\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":25,\"w\":13,\"h\":10,\"i\":\"687597e3-4848-4629-8b85-45c0773efb79\"},\"panelIndex\":\"687597e3-4848-4629-8b85-45c0773efb79\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_6\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":35,\"w\":24,\"h\":15,\"i\":\"0174654c-2010-463a-b49e-fa5759b61b9c\"},\"panelIndex\":\"0174654c-2010-463a-b49e-fa5759b61b9c\",\"embeddableConfig\":{},\"panelRefName\":\"panel_7\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":50,\"w\":48,\"h\":21,\"i\":\"36e03a4a-e017-42b8-82cf-205d26b2ed6b\"},\"panelIndex\":\"36e03a4a-e017-42b8-82cf-205d26b2ed6b\",\"embeddableConfig\":{},\"panelRefName\":\"panel_8\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":71,\"w\":48,\"h\":21,\"i\":\"e1c0f1e0-de36-4527-bafa-a297fe9452a2\"},\"panelIndex\":\"e1c0f1e0-de36-4527-bafa-a297fe9452a2\",\"embeddableConfig\":{},\"panelRefName\":\"panel_9\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":92,\"w\":13,\"h\":20,\"i\":\"74a841b8-2ffc-4f6d-8b5a-ca7960eb6b10\"},\"panelIndex\":\"74a841b8-2ffc-4f6d-8b5a-ca7960eb6b10\",\"embeddableConfig\":{},\"panelRefName\":\"panel_10\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":92,\"w\":35,\"h\":20,\"i\":\"f15e46fe-040f-4602-ad13-01aab36b372a\"},\"panelIndex\":\"f15e46fe-040f-4602-ad13-01aab36b372a\",\"embeddableConfig\":{},\"panelRefName\":\"panel_11\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":112,\"w\":16,\"h\":17,\"i\":\"bfdc6d50-66f1-4f9a-9ea5-cd30bc01099d\"},\"panelIndex\":\"bfdc6d50-66f1-4f9a-9ea5-cd30bc01099d\",\"embeddableConfig\":{},\"panelRefName\":\"panel_12\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":16,\"y\":112,\"w\":32,\"h\":17,\"i\":\"efbd7f15-5af7-4e39-9889-c1c944a40dc2\"},\"panelIndex\":\"efbd7f15-5af7-4e39-9889-c1c944a40dc2\",\"embeddableConfig\":{\"columns\":[\"@timestamp\",\"host.name\",\"zeek.reporter.level\",\"zeek.reporter.msg\",\"zeek.reporter.location\"]},\"panelRefName\":\"panel_13\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":24,\"y\":35,\"w\":24,\"h\":15,\"i\":\"2ecc4ac3-d694-46ab-a6b1-9c86e5e9d394\"},\"panelIndex\":\"2ecc4ac3-d694-46ab-a6b1-9c86e5e9d394\",\"embeddableConfig\":{},\"panelRefName\":\"panel_14\"}]", "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", "version": 1, "timeRestore": false, @@ -468,7 +468,7 @@ "version": "Wzg5NywxXQ==", "attributes": { "title": "Zeek - Reporter Categories", - "visState": "{\"title\":\"Zeek - Reporter Categories\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.reporter.level.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100}}}", + "visState": "{\"title\":\"Zeek - Reporter Categories\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.reporter.level\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100}}}", "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", "description": "", "version": 1, @@ -504,7 +504,7 @@ "host.name", "zeek.reporter.level", "zeek.reporter.location", - "zeek.reporter.message" + "zeek.reporter.msg" ], "sort": [], "version": 1, diff --git a/dashboards/templates/composable/component/zeek_diagnostic.json b/dashboards/templates/composable/component/zeek_diagnostic.json index 9aac62141..f4b8f0420 100644 --- a/dashboards/templates/composable/component/zeek_diagnostic.json +++ b/dashboards/templates/composable/component/zeek_diagnostic.json @@ -27,7 +27,7 @@ "packet_filter.failure_reason": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, "print.vals": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, "reporter.level": { "type": "keyword" }, - "reporter.message": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "reporter.msg": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, "reporter.location": { "type": "keyword" }, "stats.peer": { "type": "keyword" }, "stats.mem": { "type": "long" }, From b0356c8cb2ebf6d87abcabb53a364c5a3b457ffa Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 5 Mar 2024 10:14:59 -0700 Subject: [PATCH 79/79] Fix regression found in testing idaholab/Malcolm#336 --- shared/bin/zeek_threat_feed_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py index 229fd0449..3716cafe1 100644 --- a/shared/bin/zeek_threat_feed_utils.py +++ b/shared/bin/zeek_threat_feed_utils.py @@ -512,7 +512,9 @@ def ProcessMISP( certainty = None # determine if we're processing an event or an attribute - if 'info' in toParse: + if (('Event' in toParse) and isinstance(toParse['Event'], dict) and ('info' in toParse['Event'])) or ( + 'info' in toParse + ): # this is an event, which may contain an array of attributes event = MISPEvent() event.from_dict(**toParse)