Showing exemplars on Prometheus

Hi All,

I am trying to use prometheus to display data with exemplars to show traceID but I cannot seem to get it to display the exemplars

I used this repo as a base and placed in prometheus and grafana tempo.

https://github.com/TamimiGitHub/solace-dt-demo

#Docker Compose
version: '3.5'
networks:

solace_msg_net:

name: solace_msg_net

external: false

driver: bridge
services:
#Prometheus

prometheus:

container_name: prometheus

image: prom/prometheus

networks:

- solace_msg_net

restart: always

command:

- --config.file=/etc/prometheus/prometheus.yaml

- --enable-feature=exemplar-storage #Enable exemplar storage

volumes:

- ./prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml

ports:

- "9090:9090"
Tempo
tempo:

image: grafana/tempo:latest

restart: always

command: ["-config.file=/etc/tempo.yaml"]

networks:

- solace_msg_net

container_name: tempo1

hostname: tempo

volumes:

- ./tempo/tempo-config.yaml:/etc/tempo.yaml

ports:

- "3200:3200"

- "4317:4317"

healthcheck:

interval: 5s

retries: 10

test: wget --no-verbose --tries=1 --spider http://localhost:3200/status || exit 1
Collector
otel-collector:

image: ${OTELCOL_IMG}

command: ["--config=/etc/otel-collector-config.yaml", "${OTELCOL_ARGS}"]

hostname: otelcollector

networks:

- solace_msg_net

volumes:

- ./otel-collector/otel-collector-config.yaml:/etc/otel-collector-config.yaml

ports:

- "1888:1888"   # pprof extension

- "8888:8888"   # Prometheus metrics exposed by the collector

- "8889:8889"   # Prometheus exporter metrics

- "13133:13133" # health_check extension

- "55679:55679" # zpages extension
solbroker:

image: $PUBSUB_IMAGE

hostname: $PUBSUB_HOSTNAME

networks:

- solace_msg_net

env_file:

- ./solace_config_keys.env

ports:

- "2222:2222"

- "8080:8080"

- "55443:55443"

- "55445:55445"

- "${PUBSUB_PORT_NUMBER}:55555"

- "55556:55556"

- "5671:5671"

- "5672:5672"

- "5550:5550"

- "1943:1943"

- "8008:8008"

shm_size: 2g

ulimits:

memlock: -1

nofile:

soft: 2448

hard: 42192
solace_prometheus_exportor:

image: solacecommunity/solace-prometheus-exporter:latest

container_name: prometheus-exporter

networks:

- solace_msg_net

environment:

- SOLACE_LISTEN_ADDR=0.0.0.0:9628

- SOLACE_SCRAPE_URI=http://solbroker:8080

- SOLACE_USERNAME=admin

- SOLACE_PASSWORD=admin

- SOLACE_TIMEOUT=5s

- SOLACE_DEFAULT_VPN=default

- SOLACE_SSL_VERIFY=false

- SOLACE_LISTEN_TLS=false
ports:
  - 9628:9628
  
  
grafana:

image: grafana/grafana

container_name: grafana1

hostname: grafana

networks:

- solace_msg_net

depends_on:

tempo:

condition: service_healthy

volumes:

- ./grafana/grafana-bootstrap.ini:/etc/grafana/grafana.ini

- ./grafana/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml

environment:

- GF_AUTH_ANONYMOUS_ENABLED=true

- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin

- GF_AUTH_DISABLE_LOGIN_FORM=true

ports:

- "3000:3000"

healthcheck:

interval: 5s

retries: 10

test: wget --no-verbose --tries=1 --spider http://localhost:3000 || exit 1

#Otel Collector
receivers:

  otlp:

    protocols:

      grpc:
  solace:

    broker: [solbroker:5672]

    max_unacknowledged: 500

    auth:

      sasl_plain:

        username: trace

        password: trace

     queue: queue://#telemetry-trace

     tls:

      insecure: true

      insecure_skip_verify: true
processors:

    memory_limiter:

      check_interval: 1s

      limit_mib: 1000

      spike_limit_mib: 500

  batch:
exporters:
      prometheus:

      endpoint: localhost:9090

      enable_open_metrics: true
    otlp:

      endpoint: tempo:4317

    tls:

      insecure: true
service:

  pipelines:

    metrics:

      receivers: [otlp]

      processors: [batch]

      exporters: [prometheus]
    traces:
      receivers: [solace, otlp]
      processors: [batch, memory_limiter]
      exporters: [otlp]

#Prometheus.yaml

global:

scrape_interval: 10s

evaluation_interval: 10s
scrape_configs:
job_name: 'tempo'

static_configs:targets: ['tempo:3200']job_name: 'otel-collector'

metrics_path: /metrics

static_configs:targets: ['otel-collector:8888']job_name: 'solace'

metrics_path: /solace-det

static_configs:targets: ['solace_prometheus_exportor:9628']

I am wondering if I missed out something.

Thanks.