> ## Documentation Index
> Fetch the complete documentation index at: https://seilabs-docs-bridge-release-v6-7-0.mintlify.site/llms.txt
> Use this file to discover all available pages before exploring further.

# Sei Node Advanced Configuration & Monitoring

> Optimize your Sei node's performance with advanced system configurations, monitoring setup with Prometheus and Grafana, and effective alerting strategies for maintaining reliable node operations.

## Optimizing system configuration

The number of possible setups is virtually unlimited, so this document cannot cover them all. Even similar builds and configurations can behave differently because of external factors. Your results may vary.

Use these general guidelines as a starting point. Be cautious and make incremental changes. Test and observe each change before you move forward.

Always focus on only one specific area at a time. Do not change the memory, storage, and CPU configurations all at once. Otherwise, it becomes nearly impossible to diagnose problems.

### Memory management

These settings in `/etc/sysctl.conf` can optimize memory usage and disk I/O patterns:

```bash theme={"dark"}
# Minimize swapping
vm.swappiness = 1

# Control disk write behavior
vm.dirty_background_ratio = 3
vm.dirty_ratio = 10
vm.dirty_expire_centisecs = 300
vm.dirty_writeback_centisecs = 100
```

To apply the changes, run `sudo sysctl -p`.

### Network stack

These settings in `/etc/sysctl.conf` may improve network performance:

```bash theme={"dark"}
# Increase connection handling capacity
net.core.somaxconn = 32768
net.core.netdev_max_backlog = 32768
net.ipv4.tcp_max_syn_backlog = 16384

# Optimize buffer sizes
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 87380 16777216
```

### Storage configuration

For NVMe drives, optimize I/O scheduling:

Storage optimization commands

```bash theme={"dark"}
# Set IO scheduler
echo "none" > /sys/block/nvme0n1/queue/scheduler

# Set read-ahead buffer
blockdev --setra 4096 /dev/nvme0n1

# Set IO priority in systemd service
sudo tee -a /etc/systemd/system/seid.service << EOF
[Service]
IOSchedulingClass=realtime
IOSchedulingPriority=2
EOF

# Configure disk mount options
sudo tee -a /etc/fstab << EOF
/dev/nvme0n1p1 /data ext4 defaults,noatime,nosuid,nodev,noexec,commit=60 0 0
EOF
```

## Infrastructure monitoring

Monitoring is one of the most critical components of network infrastructure. This page covers monitoring, performance tuning, and alerting configuration for Cosmos SDK and Tendermint nodes.

### Prometheus setup

First, install Prometheus:

```bash theme={"dark"}
wget https://github.com/prometheus/prometheus/releases/download/v2.42.0/prometheus-2.42.0.linux-amd64.tar.gz
tar xvf prometheus-2.42.0.linux-amd64.tar.gz
```

This is an example Prometheus configuration:

```yaml theme={"dark"}
global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  - job_name: 'sei_node'
    static_configs:
      - targets: ['node1_ip:port']
    metrics_path: /metrics
  - job_name: 'node'
    static_configs:
      - targets: ['node2_ip:port']
```

### Grafana integration

Install and configure Grafana:

```bash theme={"dark"}
sudo apt install -y apt-transport-https software-properties-common
sudo add-apt-repository "deb https://packages.grafana.com/oss/deb stable main"
sudo apt update && sudo apt-get install grafana
```

<Accordion title="Sample Grafana dashboard JSON">
  ```json theme={"dark"}
  {
  	"annotations": {
  		"list": [
  			{
  				"builtIn": 1,
  				"datasource": "-- Grafana --",
  				"enable": true,
  				"hide": true,
  				"iconColor": "rgba(0, 211, 255, 1)",
  				"name": "Annotations & Alerts",
  				"type": "dashboard"
  			}
  		]
  	},
  	"editable": true,
  	"gnetId": null,
  	"graphTooltip": 0,
  	"id": 1,
  	"links": [],
  	"panels": [
  		{
  			"alerting": {},
  			"aliasColors": {},
  			"bars": false,
  			"dashLength": 10,
  			"dashes": false,
  			"datasource": null,
  			"fieldConfig": {
  				"defaults": {
  					"custom": {}
  				},
  				"overrides": []
  			},
  			"fill": 1,
  			"fillGradient": 0,
  			"gridPos": {
  				"h": 8,
  				"w": 12,
  				"x": 0,
  				"y": 0
  			},
  			"hiddenSeries": false,
  			"id": 2,
  			"legend": {
  				"avg": false,
  				"current": false,
  				"max": false,
  				"min": false,
  				"show": true,
  				"total": false,
  				"values": false
  			},
  			"lines": true,
  			"linewidth": 1,
  			"nullPointMode": "null",
  			"options": {
  				"alertThreshold": true
  			},
  			"percentage": false,
  			"pluginVersion": "7.2.0",
  			"pointradius": 2,
  			"points": false,
  			"renderer": "flot",
  			"seriesOverrides": [],
  			"spaceLength": 10,
  			"stack": false,
  			"steppedLine": false,
  			"targets": [
  				{
  					"expr": "tendermint_consensus_height",
  					"interval": "",
  					"legendFormat": "",
  					"refId": "A"
  				}
  			],
  			"thresholds": [],
  			"timeRegions": [],
  			"title": "Block Height",
  			"tooltip": {
  				"shared": true,
  				"sort": 0,
  				"value_type": "individual"
  			},
  			"type": "graph",
  			"xaxis": {
  				"buckets": null,
  				"mode": "time",
  				"name": null,
  				"show": true,
  				"values": []
  			},
  			"yaxes": [
  				{
  					"format": "short",
  					"label": null,
  					"logBase": 1,
  					"max": null,
  					"min": null,
  					"show": true
  				},
  				{
  					"format": "short",
  					"label": null,
  					"logBase": 1,
  					"max": null,
  					"min": null,
  					"show": true
  				}
  			],
  			"yaxis": {
  				"align": false,
  				"alignLevel": null
  			}
  		}
  	],
  	"schemaVersion": 26,
  	"style": "dark",
  	"tags": [],
  	"templating": {
  		"list": []
  	},
  	"time": {
  		"from": "now-6h",
  		"to": "now"
  	},
  	"timepicker": {},
  	"timezone": "",
  	"title": "Sei Node Metrics",
  	"uid": "sei_metrics",
  	"version": 1
  }
  ```
</Accordion>

### Alert management

Install Alertmanager:

```bash theme={"dark"}
wget https://github.com/prometheus/alertmanager/releases/download/v0.25.0/alertmanager-0.25.0.linux-amd64.tar.gz
tar xvf alertmanager-0.25.0.linux-amd64.tar.gz
```

<Accordion title="Create alert rules configuration">
  ```yaml theme={"dark"}
  groups:
    - name: validator_alerts
      rules:
        - alert: NodeDown
          expr: up == 0
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: 'Node {{ $labels.instance }} down'

        - alert: BlockProductionSlow
          expr: rate(tendermint_consensus_height[5m]) < 0.1
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: 'Block production is slow on {{ $labels.instance }}'
        - alert: ValidatorMissedBlocks
          expr: increase(tendermint_consensus_validator_missed_blocks[1h]) > 0
          labels:
            severity: critical
          annotations:
            summary: 'Validator missing blocks'

        - alert: ValidatorJailed
          expr: tendermint_consensus_validator_status == 0
          labels:
            severity: critical
          annotations:
            summary: 'Validator has been jailed'

        - alert: ConsensusStalled
          expr: tendermint_consensus_height_status == 0
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: 'Consensus has stalled'
  ```
</Accordion>

## Log management

### Loki setup

Use Loki for log aggregation:

```bash theme={"dark"}
wget https://github.com/grafana/loki/releases/download/v2.8.0/loki-linux-amd64.zip
unzip loki-linux-amd64.zip
```

<Accordion title="Promtail configuration">
  ```yaml theme={"dark"}
  server:
    http_listen_port: 9080

  positions:
    filename: /tmp/positions.yaml

  clients:
    - url: http://localhost:3100/loki/api/v1/push

  scrape_configs:
    - job_name: sei_logs
      static_configs:
        - targets:
            - localhost
          labels:
            job: seid_logs
            __path__: /var/log/seid/*.log
  ```
</Accordion>

### Log rotation

Configure logrotate to manage log files:

```bash theme={"dark"}
sudo tee /etc/logrotate.d/sei << EOF
/var/log/sei/*.log {
    daily
    rotate 14
    compress
    delaycompress
    notifempty
    create 0640 sei sei
    sharedscripts
    postrotate
        systemctl reload seid
    endscript
}
EOF
```

## Security configuration

### Network security

Configure the UFW firewall:

```bash theme={"dark"}
sudo ufw default deny incoming
sudo ufw default allow outgoing
sudo ufw allow 26656/tcp comment 'Sei P2P'
sudo ufw allow 26657/tcp comment 'Sei RPC'
sudo ufw allow 9090/tcp comment 'Sei gRPC'
sudo ufw enable
```

### Rate limiting

<Accordion title="Example Nginx configuration with rate limiting">
  ```nginx theme={"dark"}
  http {
      limit_req_zone $binary_remote_addr zone=sei_rpc:10m rate=10r/s;

      server {
          listen 26657;
          location / {
              limit_req zone=sei_rpc burst=20 nodelay;
              proxy_pass http://localhost:26657;
          }
      }
  }
  ```
</Accordion>

## Validator-specific monitoring

### Status query

Query the validator status through the SDK:

```bash theme={"dark"}
seid query staking validator $(seid keys show --bech val -a <validator_keyfile_name>)
```

Query the status through the REST API:

```sh theme={"dark"}
curl -s "http://localhost:1317/cosmos/staking/v1beta1/validators/<valoper_address>"
```

<Accordion title="Validator &#x22;status&#x22; query script">
  ```sh theme={"dark"}
  #!/bin/bash

  MONIKER="$1"
  API_URL="http://localhost:1317/cosmos/staking/v1beta1/validators?pagination.limit=500"

  echo "Querying validators from $API_URL..."

  VALIDATOR_DATA=$(curl -s "$API_URL" | jq -c --arg MONIKER "$MONIKER" '.validators[] | select(.description.moniker == $MONIKER)')

  if [[ -z "$VALIDATOR_DATA" ]]; then
      echo "❌ No validator found with moniker: $MONIKER"
      exit 1
  fi

  echo "Validator details:"
  echo "$VALIDATOR_DATA" | jq '.'
  ```
</Accordion>

### Critical metrics

Monitor these validator-specific metrics:

```bash theme={"dark"}
# Check signing status
seid query slashing signing-info $(seid tendermint show-validator)

# Check current delegations
seid query staking delegations-to $(seid keys show -a $VALIDATOR_KEY)
```

## Backup management

<Accordion title="Complete automated backup script">
  ```bash theme={"dark"}
  #!/bin/bash
  BACKUP_DIR="/backup/sei"
  DATE=$(date +%Y%m%d)
  NODE_HOME="/root/.sei"

  # Create backup directory
  mkdir -p $BACKUP_DIR

  # Stop service
  systemctl stop seid

  # Backup configuration
  tar czf $BACKUP_DIR/sei-config-$DATE.tar.gz $NODE_HOME/config

  # Backup data directory
  tar czf $BACKUP_DIR/sei-data-$DATE.tar.gz $NODE_HOME/data

  # Backup key files
  tar czf $BACKUP_DIR/sei-keys-$DATE.tar.gz $NODE_HOME/keyring-file

  # Start service
  systemctl start seid

  # Remove backups older than 7 days
  find $BACKUP_DIR -type f -mtime +7 -name '*.tar.gz' -delete

  # Log backup completion
  echo "Backup completed successfully on $(date)" >> $BACKUP_DIR/backup.log
  ```
</Accordion>

## Host system monitoring

### Resource usage tracking

Install and configure node\_exporter:

```bash theme={"dark"}
wget https://github.com/prometheus/node_exporter/releases/download/v1.5.0/node_exporter-1.5.0.linux-amd64.tar.gz
tar xvf node_exporter-1.5.0.linux-amd64.tar.gz
```

Add node\_exporter to the Prometheus configuration:

```yaml theme={"dark"}
scrape_configs:
  - job_name: 'node'
    static_configs:
      - targets: ['localhost:9100']
```

## Seed node Prometheus metrics

Seed nodes now expose a Prometheus `/metrics` endpoint, so they can be scraped like any other node. Because a seed serves no RPC, this endpoint is the only observability surface it has — peer reachability and peer headroom both come from its p2p metrics (for example `tendermint_p2p_peers`).

To enable it, set the usual instrumentation fields in `config.toml` under `[instrumentation]`:

```toml theme={"dark"}
[instrumentation]
prometheus = true
prometheus-listen-addr = ":26660"
```

Every exposed series is stamped with the node's `chain_id` label, matching the behavior of full nodes.

<Warning>
  For a seed node, a failure to bind the Prometheus listener (for example, the port is already in use or the address is unparseable) is **fatal**: the seed fails to start. This is deliberate — a seed's value is only realized when it is observable, so an unambiguous crash is preferable to a seed that discovers peers while blind.

  A full node behaves differently: a bind failure is **non-fatal**. The full node logs the error and continues running, because it still serves RPC as an in-band way to observe its health.
</Warning>

The `tendermint_p2p_peers` gauge is present from node startup rather than appearing only after the first refresh interval, so an alert comparing peer count against the connection cap matches immediately rather than sitting unmatched during the startup window.

### Metrics server timeouts

The Prometheus metrics HTTP server enforces request timeouts to bound how long a slow reader can hold a request slot:

| Timeout | Value |
| - | - |
| `ReadHeaderTimeout` | 10s |
| `ReadTimeout` | 10s |
| `WriteTimeout` | 30s |
| `IdleTimeout` | 60s |

### `max-open-connections` behavior

The `max-open-connections` field under `[instrumentation]` is not a socket limit despite the name. It becomes promhttp's `MaxRequestsInFlight`: the maximum number of scrapes served concurrently. Requests past the limit receive an HTTP `503` rather than being queued, and a slow reader occupies a slot until it finishes or hits the `WriteTimeout`. A value of `0` means unlimited.

Keep this value above the number of scrapers that can overlap — for example, an HA Prometheus pair, blackbox probes, and a human running `curl` — so a burst of concurrent scrapes does not cause a healthy node to report as unscrapeable.

## Cosmos SDK / Tendermint telemetry chain\_id label

Node telemetry automatically injects a `chain_id` global label into the Cosmos SDK / Tendermint Prometheus telemetry (the `GlobalLabels` of the `[telemetry]` configuration) based on the node's client chain ID. The label is appended only when a `chain_id` label is not already configured in `GlobalLabels`, so an explicit operator-set value always takes precedence. This is distinct from the `chain_id` constant label applied to the OpenTelemetry `sei_chain` namespace described above — it covers the separate Cosmos SDK/Tendermint telemetry path.

Because every emitted series carries this label, update any PromQL that aggregates across series (for example `sum(...) without (...)` or `by (...)` clauses) to account for `chain_id`, and add a `chain_id` match where you want to scope a query to a single chain — useful when one Prometheus instance scrapes nodes on more than one chain.

### New Tendermint internal metrics subsystems

Additional Prometheus metrics are exported under the `tendermint` namespace in these new subsystems:

| Subsystem prefix | Description |
| - | - |
| `tendermint_internal_autobahn_avail_*` | Autobahn availability metrics, including the commit and app road index, the commit and app global block number, proposal-to-commit latency, and commit-to-commit latency (the latter labeled by `timeouts`). |
| `tendermint_internal_autobahn_data_*` | Autobahn data latency metrics (a `latency` histogram labeled by `resource` and `stage`) tracking resource processing from production to the given stage. |
| `tendermint_internal_p2p_mux_*` | p2p mux stream metrics, including stream `latency`, `in_flight` open streams, and per-stream `send_msgs`, `recv_msgs`, `send_bytes`, and `recv_bytes` counters (labeled by `role` and `kind`). |

## EVM RPC OpenTelemetry metrics

The EVM RPC layer emits OpenTelemetry metrics through the process-wide `MeterProvider` (for example, a Prometheus exporter). It emits these metrics in parallel with the legacy `sei_*` metrics, so you can migrate dashboards incrementally.

<Note>
  Every OpenTelemetry metric series exported through the Prometheus exporter (the `sei_chain` namespace, including the `evmrpc_*`, `flatkv_*`, `litt_*`, and `pebble_*` metrics documented below) now carries a `chain_id` constant label derived from the node's chain ID. The label is applied to every emitted series, so dashboards and alerting queries can group or filter by chain — for example when a single Prometheus instance scrapes nodes on more than one chain. Update any PromQL that aggregates across series (for example `sum(...) without (...)` or `by (...)` clauses) to account for the new label, and add a `chain_id` match where you want to scope a query to a single chain.
</Note>

### Available EVM RPC metrics

| Metric | Type | Description |
| - | - | - |
| `evmrpc_request_latency_seconds` | Histogram | EVM RPC request latency in seconds. |
| `evmrpc_websocket_connects_total` | Counter | Number of new WebSocket connections. |
| `evmrpc_redirected_requests_total` | Counter | Number of EVM RPC requests forwarded to another validator. Labeled by `endpoint` and `connection`. |
| `evmrpc_historical_debug_trace_attempts_total` | Counter | Number of `debug_trace*` requests that target historical blocks beyond the configured max block lookback. Labeled by `endpoint` and `connection`. |
| `evmrpc_requests_rejected_total` | Counter | Number of JSON-RPC requests rejected by admission control. Labeled by `protocol` (`http` or `ws`) and `reason` (`oversize` or `busy`). On HTTP (:8545), an `oversize` request body exceeds the configured `max_request_body_bytes` cap and is rejected with HTTP 413, while `busy` means the `max_concurrent_request_bytes` budget is exhausted and the request is rejected with HTTP 429. On WebSocket (:8546), `oversize` frames exceed `max_request_body_bytes` and close the connection with WebSocket close code 1009 (no JSON-RPC error response), while `busy` means the connection waited for concurrent-byte budget longer than `ws_admission_timeout` and the peer receives JSON-RPC error `-32005` before the connection is closed (#3818). |

The `evmrpc_request_latency_seconds` histogram carries these labels:

| Label | Description |
| - | - |
| `endpoint` | The RPC method that the node serves (for example, `eth_getBalance`). |
| `connection` | The connection type that serves the request (for example, `http` or `websocket`). |
| `success` | Boolean that indicates whether the request succeeded. |
| `error_class` | A low-cardinality classification of the failure. An empty string means success. Possible values include `panic`, `execution_reverted`, `evm_not_supported`, `sei_legacy_disabled`, `association_missing`, `jsonrpc_error`, and `unknown`. |
| `jsonrpc_code` | A low-cardinality bucket for the JSON-RPC error code: `spec` (predefined range `-32700..-32600`), `server` (server-defined range `-32099..-32000`), or `other`. An empty string means no code (success or an untyped error). |

### Migrating from legacy EVM RPC metrics

These legacy `sei_*` metrics remain available today but are deprecated. They are scheduled for removal after dashboards migrate to the `evmrpc_*` OpenTelemetry metrics:

| Legacy metric | Replacement |
| - | - |
| `sei_rpc_request_latency_ms` | `evmrpc_request_latency_seconds` |
| `sei_websocket_connect` | `evmrpc_websocket_connects_total` |
| `sei_rpc_request_counter` | `evmrpc_request_latency_seconds` (use the histogram count with the `success` and `error_class` labels) |

Before the legacy metrics are removed, update your Prometheus and Grafana dashboards to use the `evmrpc_*` metrics. The latency unit changed from milliseconds (`sei_rpc_request_latency_ms`) to seconds (`evmrpc_request_latency_seconds`). Adjust any thresholds and panel formatting to match.

## FlatKV OpenTelemetry metrics

The FlatKV state store emits OpenTelemetry metrics through the process-wide `MeterProvider` (for example, a Prometheus exporter). With these metrics, you can observe commit throughput, catchup progress, snapshotting, rollbacks, and snapshot imports.

### Available FlatKV metrics

| Metric | Type | Description |
| - | - | - |
| `flatkv_open_latency` | Histogram | Time taken to open the FlatKV store (seconds). |
| `flatkv_apply_changesets_latency` | Histogram | Time taken to apply changesets to FlatKV (seconds). |
| `flatkv_commit_latency` | Histogram | Time taken to commit FlatKV changes (seconds). |
| `flatkv_commit_batch_latency` | Histogram | Time taken to commit a FlatKV data DB batch (seconds). |
| `flatkv_batch_read_old_values_latency` | Histogram | Time taken to batch read old FlatKV values (seconds). |
| `flatkv_num_kv_pairs` | Counter | Number of key-value pairs applied to FlatKV. |
| `flatkv_pending_writes` | Gauge | Current number of pending FlatKV writes. |
| `flatkv_current_version` | Gauge | Current committed FlatKV version. |
| `flatkv_catchup_latency` | Histogram | Time taken to replay FlatKV WAL entries (seconds). |
| `flatkv_catchup_replay_num_blocks` | Counter | Number of FlatKV WAL entries replayed during catchup. |
| `flatkv_snapshot_write_latency` | Histogram | Time taken to write a FlatKV snapshot (seconds). |
| `flatkv_snapshot_prune_latency` | Histogram | Time taken to prune FlatKV snapshots (seconds). |
| `flatkv_snapshot_prune_attempts` | Counter | Total number of FlatKV snapshot prune attempts. |
| `flatkv_current_snapshot_height` | Gauge | Current FlatKV snapshot height. |
| `flatkv_rollback_latency` | Histogram | Time taken to roll back FlatKV state (seconds). |
| `flatkv_import_latency` | Histogram | Time taken to import FlatKV snapshot data (seconds). |
| `flatkv_import_kv_pairs` | Counter | Number of key-value pairs imported into FlatKV. |
| `flatkv_import_worker_flush_latency` | Histogram | Time taken to flush a FlatKV import worker batch (seconds). |
| `flatkv_flush_latency` | Histogram | Time taken to flush a FlatKV data DB (seconds). |

### Labels

FlatKV metrics carry these labels where applicable:

| Label | Description |
| - | - |
| `db` | The data DB that the measurement applies to (for example, `account`, `storage`, `code`, or `legacy`). Present on per-DB metrics such as `flatkv_commit_batch_latency`, `flatkv_flush_latency`, `flatkv_num_kv_pairs`, `flatkv_pending_writes`, `flatkv_import_kv_pairs`, and `flatkv_import_worker_flush_latency`. |
| `success` | Boolean that indicates whether the operation succeeded. Present on latency and attempt metrics that can fail. |
| `read_only` | Boolean, present on `flatkv_open_latency`, that indicates whether the store was opened read-only. |

### Enabling Pebble internal metrics

A single FlatKV-level knob controls Pebble's internal (per-DB) metrics: `enable-pebble-metrics` under `[state-commit.flatkv]` in `app.toml` (default `true`). The node honors this key when it is present, but the `app.toml` that `seid init` generates does not include it. To change the default, add the key manually. The value propagates to every data DB (account, code, storage, legacy, and metadata) during initialization. It overrides any per-DB `EnableMetrics` settings. Configure Pebble metrics through this knob, not through the individual per-DB settings.

## Pebble estimated read/write metrics

SeiDB can emit lightweight, opt-in estimates of logical read and write operations observed by its Pebble-backed wrappers. These counters are disabled by default and are intended to approximate read amplification (estimated reads divided by estimated writes) without the overhead of Pebble's full internal metrics.

### Available metrics

| Metric | Type | Unit | Description |
| - | - | - | - |
| `pebble_estimated_reads` | Counter | `{count}` | Estimated logical PebbleDB reads observed by SeiDB wrappers. |
| `pebble_estimated_writes` | Counter | `{count}` | Estimated logical PebbleDB writes observed by SeiDB wrappers. |

Both counters carry a `db` attribute identifying the database the measurement applies to (derived from the base name of the data directory), so you can distinguish FlatKV data DBs, the state-store backend, and receipt storage.

### Enabling the counters

These estimates are controlled per subsystem in `app.toml`. All default to `false`:

| Config key | Section | Description |
| - | - | - |
| `enable-read-write-metrics` | `[state-commit.flatkv]` | Emits estimated read/write counters for FlatKV's Pebble DBs. |
| `ss-enable-read-write-metrics` | `[state-store]` | Emits estimated PebbleDB MVCC read/write counters for the state-store backend. Applies when `ss-backend = "pebbledb"`. |
| `enable-read-write-metrics` | `[receipt-store]` | Emits estimated read/write counters for Pebble-backed receipt storage. |

For example, to enable the FlatKV counters, add the following to `app.toml`:

```toml theme={"dark"}
[state-commit.flatkv]
enable-read-write-metrics = true
```

## Tuning littidx eth\_getLogs parallelism

When the receipt store uses the `littidx` backend, each `eth_getLogs` query scans the requested block range across a bounded worker pool instead of one block at a time. Per-block tag index scans and litt body reads are independent, so fanning them across multiple workers reduces latency on wide-range queries while results stay in strict `(block, txIndex)` order.

The `log-filter-parallelism` config key under `[receipt-store]` in `app.toml` bounds how many blocks a single query scans concurrently. It defaults to `16` and applies only to the `littidx` backend. A value `<= 0` falls back to the default.

```toml theme={"dark"}
[receipt-store]
log-filter-parallelism = 16
```

## Per-block hash logger

The state-commit store can record a per-block CSV of named block hashes to disk as a debugging and forensics tool. For each committed block it logs the memIAVL per-module and root hashes, the flatKV per-DB and root hashes, the application hash, the Tendermint block hash, the result hash (the merkle root over the block's deterministic transaction results, equal to the next block's `LastResultsHash`), and the changeset hash. Recording the same hashes under one block number on every node makes it straightforward to pinpoint where and when block-hash computation diverges across nodes.

The feature is **enabled by default**. By default it writes files into a `hash.log` directory under the state-commit store's data directory (that is, `<home>/data/hash.log`).

### Configuration

The hash logger is configured with five `app.toml` fields under `[state-commit]`:

| Config key | Default | Description |
| - | - | - |
| `sc-hash-logger-enable` | `true` | Turns per-block hash logging on or off. |
| `sc-hash-logger-directory` | empty | Directory for hash log files. When empty, defaults to `<home>/data/hash.log`. |
| `sc-hash-logger-blocks-to-retain` | `0` | Number of most-recent blocks to keep on disk. `0` disables block-count retention (the disk-size cap is then the only bound). |
| `sc-hash-logger-target-file-size` | `16 MB` | Size in bytes a log file may reach before it is sealed and a new one is opened. Must be greater than `0`. |
| `sc-hash-logger-max-disk-size` | `16 GB` | Backstop cap in bytes on the total size of sealed log files. `0` disables the disk-size cap (block-count retention is then the only bound). |

Retention is disk-driven by default: up to roughly 16 GiB of sealed files are kept, with block-count retention disabled. If both `sc-hash-logger-blocks-to-retain` and `sc-hash-logger-max-disk-size` are set to `0`, no retention bound applies and the logs grow without limit — a deliberate operator choice.

For example, to disable the hash logger, add the following to `app.toml`:

```toml theme={"dark"}
[state-commit]
sc-hash-logger-enable = false
```

## LittDB OpenTelemetry metrics

LittDB emits its metrics through the process-wide OpenTelemetry `MeterProvider` instead of a private Prometheus client. When `MetricsEnabled` is set, LittDB records its metrics into the global provider. How those metrics are exported depends on `MetricsServeEndpoint` (default `false`):

* When `MetricsServeEndpoint` is `false` (the default), LittDB records into the already-configured global `MeterProvider` and leaves exporting to the embedding application. The application is responsible for standing up the Prometheus exporter and serving the registry. No exporter or `/metrics` server is created by LittDB, and `MetricsPort` is ignored.
* When `MetricsServeEndpoint` is `true`, LittDB stands up its own Prometheus exporter on the global provider and serves `/metrics` on `MetricsPort` (default `9101`).

`MetricsPort` is ignored unless both `MetricsEnabled` and `MetricsServeEndpoint` are `true`. The `MetricsNamespace` and `MetricsRegistry` config fields were removed, and all metric names use a fixed `litt_` prefix.

### Available LittDB metrics

| Metric | Type | Unit | Description |
| - | - | - | - |
| `litt_table_size_bytes` | Gauge | bytes | The size of individual tables in the database. |
| `litt_table_key_count` | Gauge | count | The number of keys in individual tables in the database. |
| `litt_open_iterator_count` | Gauge | count | The number of currently open iterators for individual tables in the database. A persistently nonzero value indicates a leaked iterator. Garbage collection proceeds while iterators are open — an iterator pins its snapshot segments via reservations so their files survive until it closes — so a leaked iterator pins those segment files on disk indefinitely rather than suspending garbage collection. |
| `litt_bytes_read` | Counter | bytes | The number of bytes read from disk since startup. |
| `litt_keys_read` | Counter | count | The number of keys read from disk since startup. |
| `litt_cache_hits` | Counter | count | The number of cache hits since startup. |
| `litt_cache_misses` | Counter | count | The number of cache misses since startup. |
| `litt_read_latency_seconds` | Histogram | seconds | Read latency of the database, including both cache hits and cache misses. |
| `litt_cache_miss_latency_seconds` | Histogram | seconds | Read latency measured only when a cache miss occurs. |
| `litt_bytes_written` | Counter | bytes | The number of bytes written to disk since startup (values only, not metadata). |
| `litt_keys_written` | Counter | count | The number of keys written to disk since startup. |
| `litt_write_latency_seconds` | Histogram | seconds | Write latency of the database. |
| `litt_flush_count` | Counter | count | The number of times a flush operation has been performed. |
| `litt_flush_latency_seconds` | Histogram | seconds | Latency of a flush operation. |
| `litt_segment_flush_latency_seconds` | Histogram | seconds | Segment flush latency, a subset of the time spent during a flush operation. |
| `litt_keymap_flush_latency_seconds` | Histogram | seconds | Keymap flush latency, a subset of the time spent during a flush operation. |
| `litt_garbage_collection_latency_seconds` | Histogram | seconds | Latency of garbage collection operations. |
| `litt_chunk_cache_key_count` | Gauge | count | The number of keys in the chunk cache. |
| `litt_chunk_cache_weight_bytes` | Gauge | bytes | The weight of the chunk cache in bytes. |
| `litt_chunk_cache_keys_added` | Counter | count | The number of keys added to the chunk cache. |
| `litt_chunk_cache_weight_added_bytes` | Counter | bytes | The weight of the entries added to the chunk cache. |
| `litt_chunk_cache_eviction_latency_seconds` | Histogram | seconds | Eviction latency of the chunk cache. |

\| `litt_compression_latency_seconds` | Histogram | seconds | Latency of compressing a batch of values before they are written. Emitted only for tables with value compression enabled. |
\| `litt_compression_uncompressed_bytes` | Counter | bytes | The number of uncompressed value bytes submitted to compression since startup. |
\| `litt_compression_compressed_bytes` | Counter | bytes | The number of compressed value bytes produced by compression since startup. Compared against `litt_compression_uncompressed_bytes`, this gives the aggregate compression ratio and total bytes saved. |
\| `litt_compression_ratio` | Histogram | ratio | The per-batch compression ratio (compressed bytes divided by uncompressed bytes); lower is better. |

### Attributes

| Attribute | Description |
| - | - |
| `table` | The table that the observation applies to. Present on per-table metrics such as `litt_bytes_read`, `litt_read_latency_seconds`, `litt_table_size_bytes`, and the flush/GC latency histograms. |
| `cache` | The cache instance that the observation applies to (`chunk_read` or `chunk_write`). Present on the `litt_chunk_cache_*` metrics, which distinguish read and write caches by this attribute instead of by separate metric names. |

### Migrating from legacy LittDB metrics

The OpenTelemetry migration changed the metric names, units, and shape. You must update your existing Prometheus and Grafana dashboards:

* Latency metrics moved from millisecond summaries (for example, `{namespace}_read_latency_ms`) to second histograms (`litt_read_latency_seconds`). Change thresholds and panel formatting from milliseconds to seconds.
* Counters and gauges gained a fixed `litt_` prefix and explicit units. For example, `bytes_read` became `litt_bytes_read`, and the cache weight gauge became `litt_chunk_cache_weight_bytes`.
* The per-cache series that previously had separate metric names (for example, `chunk_read_cache_*` and `chunk_write_cache_*`) are now the shared `litt_chunk_cache_*` metrics. The `cache` attribute distinguishes them.
* The `MetricsNamespace` and `MetricsRegistry` config fields no longer exist. Metric names are fixed, and the global OTel provider always backs the metrics. Set the scrape port with `MetricsPort`.

## Performance testing

<Accordion title="Example benchmark script using `eth_getLogs`">
  ```js theme={"dark"}
  import { ethers } from 'ethers';

  // Configuration
  const EVM_RPC_URL = 'http://localhost:8545'; // EVM RPC endpoint to test
  const CONTRACT_ADDRESS = '0x0000000000000000000000000000000000001002'; // replace with very active contract for best results
  const INITIAL_BLOCK_RANGE = 50; // range of blocks to query using 'eth_getLogs'
  const RANGE_INCREMENT = 10; // additional blocks to query each consecutive round
  const MAX_TESTS = 50; // total number of rounds for testing

  // Store metrics for final analysis
  const metrics = [];

  function getResponseSize(logs) {
    return Buffer.byteLength(JSON.stringify(logs), 'utf8');
  }

  function formatBytes(bytes) {
    if (bytes === 0) return '0 B';
    const k = 1024;
    const sizes = ['B', 'KB', 'MB', 'GB'];
    const i = Math.floor(Math.log(bytes) / Math.log(k));
    return `${parseFloat((bytes / Math.pow(k, i)).toFixed(2))} ${sizes[i]}`;
  }

  function padString(str, length) {
    return String(str).padEnd(length);
  }

  function analyzeResults(metrics) {
    console.log('\nPerformance Analysis');
    console.log('='.repeat(50));

    // Filter out queries with no logs for meaningful statistics
    const queriesWithLogs = metrics.filter((m) => m.logsCount > 0);
    const totalQueries = metrics.length;

    console.log(`\nGeneral Statistics:`);
    console.log(`Total Queries Run: ${totalQueries}`);
    console.log(`Queries with Logs: ${queriesWithLogs.length}`);
    console.log(`Empty Responses: ${totalQueries - queriesWithLogs.length}`);

    if (queriesWithLogs.length > 0) {
      const avgResponseTime = queriesWithLogs.reduce((acc, m) => acc + m.responseTime, 0) / queriesWithLogs.length;
      const avgLogsPerQuery = queriesWithLogs.reduce((acc, m) => acc + m.logsCount, 0) / queriesWithLogs.length;
      const maxLogs = Math.max(...queriesWithLogs.map((m) => m.logsCount));
      const maxLogsQuery = queriesWithLogs.find((m) => m.logsCount === maxLogs);

      console.log(`\nPerformance Metrics:`);
      console.log(`Average Response Time (with logs): ${avgResponseTime.toFixed(2)}ms`);
      console.log(`Average Logs per Query: ${avgLogsPerQuery.toFixed(2)}`);
      console.log(`Maximum Logs in Single Query: ${maxLogs}`);
      if (maxLogsQuery) {
        console.log(`- At Range Size: ${maxLogsQuery.rangeSize} blocks`);
        console.log(`- Response Time: ${maxLogsQuery.responseTime}ms`);
        console.log(`- Efficiency: ${maxLogsQuery.logsPerMs.toFixed(3)} logs/ms`);
      }

      // Identify optimal range size based on logs/ms
      const bestEfficiency = queriesWithLogs.reduce((best, m) => (m.logsPerMs > best.logsPerMs ? m : best));
      console.log(`\nOptimal Performance:`);
      console.log(`Best Efficiency: ${bestEfficiency.logsPerMs.toFixed(3)} logs/ms`);
      console.log(`- At Range Size: ${bestEfficiency.rangeSize} blocks`);
      console.log(`- Retrieved ${bestEfficiency.logsCount} logs in ${bestEfficiency.responseTime}ms`);
    }
  }

  async function testEthGetLogs() {
    const provider = new ethers.JsonRpcProvider(EVM_RPC_URL);

    try {
      const latestBlock = await provider.getBlockNumber();
      console.log(`Latest block: ${latestBlock} (0x${latestBlock.toString(16)})`);

      let currentToBlock = latestBlock;
      let currentRange = INITIAL_BLOCK_RANGE;
      let testCount = 0;

      // Column headers with fixed widths
      console.log('\nBlock Range         Time  Logs    Size     B/ms   Logs/ms  KB/Log  Range');
      console.log('='.repeat(80));

      while (testCount < MAX_TESTS && currentToBlock > 0) {
        const fromBlock = Math.max(0, currentToBlock - currentRange);

        try {
          const startTime = Date.now();
          const filter = {
            fromBlock: fromBlock,
            toBlock: currentToBlock,
            address: CONTRACT_ADDRESS
          };

          const logs = await provider.getLogs(filter);

          const endTime = Date.now();
          const responseTime = endTime - startTime;
          const logsCount = logs.length;
          const responseSize = getResponseSize(logs);

          // Calculate metrics
          const bytesPerMs = (responseSize / responseTime).toFixed(1);
          const logsPerMs = (logsCount / responseTime).toFixed(3);
          const kbPerLog = logsCount > 0 ? (responseSize / 1024 / logsCount).toFixed(2) : 'N/A';

          // Store metrics for analysis
          metrics.push({
            rangeSize: currentRange,
            responseTime,
            logsCount,
            responseSize,
            bytesPerMs: parseFloat(bytesPerMs),
            logsPerMs: parseFloat(logsPerMs),
            kbPerLog: kbPerLog !== 'N/A' ? parseFloat(kbPerLog) : 0
          });

          // Format block range
          const rangeDisplay = `${fromBlock.toString(16)}-${currentToBlock.toString(16)}`;

          // Log with fixed column widths
          console.log(padString(rangeDisplay, 17) + padString(responseTime, 6) + padString(logsCount, 8) + padString(formatBytes(responseSize), 9) + padString(bytesPerMs, 8) + padString(logsPerMs, 9) + padString(kbPerLog, 8) + currentRange);

          if (logsCount === 10000) {
            console.log(`\nWarning: Hit 10000 log limit at range ${currentRange}`);
          }

          currentToBlock = fromBlock - 1;
          currentRange += RANGE_INCREMENT;
          testCount++;
        } catch (error) {
          console.log(`Error at range ${currentRange}: ${error.message}`);
          currentRange = Math.max(INITIAL_BLOCK_RANGE, currentRange - RANGE_INCREMENT);
          currentToBlock = fromBlock - 1;
          testCount++;
        }

        await new Promise((resolve) => setTimeout(resolve, 1000));
      }

      // Perform final analysis
      analyzeResults(metrics);
    } catch (error) {
      console.error('Failed to initialize or get latest block:', error);
      process.exit(1);
    }
  }

  // Run the test
  testEthGetLogs();
  ```
</Accordion>

For specific customizations or other metrics, ask the Sei
technical communities on [Telegram](https://t.me/+ZN-NcvOWStQwMzk0) or
[Discord](https://discord.gg/sei).


This documentation is built and hosted on [Mintlify](https://mintlify.com), a developer documentation platform.