Merge branch 'antanas/better-error-handling' into antanas/prom-remote…

…-write-backend-local-setup
m3db · Oct 1, 2021 · 94c3c68 · 94c3c68
2 parents bfc53e3 + 31e82fe
commit 94c3c68
Show file tree

Hide file tree

Showing 60 changed files with 2,673 additions and 385 deletions.
diff --git a/scripts/docker-integration-tests/carbon/m3coordinator.yml b/scripts/docker-integration-tests/carbon/m3coordinator.yml
@@ -12,6 +12,7 @@ clusters:
                 - dbnode01:2379
 
 carbon:
+  findResultsIncludeBothExpandableAndLeaf: true
   ingester:
     listenAddress: "0.0.0.0:7204"
     rewrite:

diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml b/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml
@@ -0,0 +1,108 @@
+version: "3.5"
+services:
+  m3aggregator01:
+    expose:
+      - "6001"
+    ports:
+      - "127.0.0.1:6001:6001"
+    networks:
+      - backend
+    environment:
+      - M3AGGREGATOR_HOST_ID=m3aggregator01
+    image: "m3aggregator_integration:${REVISION}"
+    volumes:
+      - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+  m3aggregator02:
+    expose:
+      - "6002"
+    ports:
+      - "127.0.0.1:6002:6001"
+    networks:
+      - backend
+    environment:
+      - M3AGGREGATOR_HOST_ID=m3aggregator02
+    image: "m3aggregator_integration:${REVISION}"
+    volumes:
+      - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+  m3coordinator01:
+    expose:
+      - "7202"
+    ports:
+      - "0.0.0.0:7202:7201"
+    networks:
+      - backend
+    image: "m3coordinator_integration:${REVISION}"
+    volumes:
+      - "./:/etc/m3coordinator/"
+  coordinatoradmin:
+    expose:
+      - "7201"
+    ports:
+      - "0.0.0.0:7201:7201"
+    networks:
+      - backend
+    image: "m3coordinator_integration:${REVISION}"
+    volumes:
+      - "./m3coordinator-admin.yml:/etc/m3coordinator/m3coordinator.yml"
+  prometheusraw:
+    expose:
+      - "9090"
+    ports:
+      - "0.0.0.0:9090:9090"
+    networks:
+      - backend
+    image: prom/prometheus:latest
+    volumes:
+      - "./prometheus.yml:/etc/prometheus/prometheus.yml"
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
+      - "--web.console.templates=/usr/share/prometheus/consoles"
+      - "--enable-feature=remote-write-receiver"
+  prometheusagg:
+    expose:
+      - "9091"
+    ports:
+      - "0.0.0.0:9091:9090"
+    networks:
+      - backend
+    image: prom/prometheus:latest
+    volumes:
+      - "./prometheus.yml:/etc/prometheus/prometheus.yml"
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
+      - "--web.console.templates=/usr/share/prometheus/consoles"
+      - "--enable-feature=remote-write-receiver"
+  etcd01:
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    networks:
+      - backend
+    image: quay.io/coreos/etcd:v3.4.3
+    command:
+      - "etcd"
+      - "--name"
+      - "etcd01"
+      - "--listen-peer-urls"
+      - "http://0.0.0.0:2380"
+      - "--listen-client-urls"
+      - "http://0.0.0.0:2379"
+      - "--advertise-client-urls"
+      - "http://etcd01:2379"
+      - "--initial-cluster-token"
+      - "etcd-cluster-1"
+      - "--initial-advertise-peer-urls"
+      - "http://etcd01:2380"
+      - "--initial-cluster"
+      - "etcd01=http://etcd01:2380"
+      - "--initial-cluster-state"
+      - "new"
+      - "--data-dir"
+      - "/var/lib/etcd"
+networks:
+  backend:
diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml
@@ -0,0 +1,216 @@
+logging:
+  level: info
+
+metrics:
+  scope:
+    prefix: m3aggregator
+  prometheus:
+    onError: none
+    handlerPath: /metrics
+    listenAddress: 0.0.0.0:6002
+    timerType: histogram
+  sanitization: prometheus
+  samplingRate: 1.0
+  extended: none
+
+m3msg:
+  server:
+    listenAddress: 0.0.0.0:6000
+    retry:
+      maxBackoff: 10s
+      jitter: true
+  consumer:
+    messagePool:
+      size: 16384
+      watermark:
+        low: 0.2
+        high: 0.5
+
+http:
+  listenAddress: 0.0.0.0:6001
+  readTimeout: 60s
+  writeTimeout: 60s
+
+kvClient:
+  etcd:
+    env: override_test_env
+    zone: embedded
+    service: m3aggregator
+    cacheDir: /var/lib/m3kv
+    etcdClusters:
+      - zone: embedded
+        endpoints:
+          - etcd01:2379
+
+runtimeOptions:
+  kvConfig:
+    environment: override_test_env
+    zone: embedded
+  writeValuesPerMetricLimitPerSecondKey: write-values-per-metric-limit-per-second
+  writeValuesPerMetricLimitPerSecond: 0
+  writeNewMetricLimitClusterPerSecondKey: write-new-metric-limit-cluster-per-second
+  writeNewMetricLimitClusterPerSecond: 0
+  writeNewMetricNoLimitWarmupDuration: 0
+
+aggregator:
+  hostID:
+    resolver: environment
+    envVarName: M3AGGREGATOR_HOST_ID
+  instanceID:
+    type: host_id
+  verboseErrors: true
+  metricPrefix: ""
+  counterPrefix: ""
+  timerPrefix: ""
+  gaugePrefix: ""
+  aggregationTypes:
+    counterTransformFnType: empty
+    timerTransformFnType: suffix
+    gaugeTransformFnType: empty
+    aggregationTypesPool:
+      size: 1024
+    quantilesPool:
+      buckets:
+        - count: 256
+          capacity: 4
+        - count: 128
+          capacity: 8
+  stream:
+    eps: 0.001
+    capacity: 32
+    streamPool:
+      size: 4096
+    samplePool:
+      size: 4096
+    floatsPool:
+      buckets:
+        - count: 4096
+          capacity: 16
+        - count: 2048
+          capacity: 32
+        - count: 1024
+          capacity: 64
+  client:
+    type: m3msg
+    m3msg:
+      producer:
+        writer:
+          topicName: aggregator_ingest
+          topicServiceOverride:
+            zone: embedded
+            environment: override_test_env
+          placement:
+            isStaged: true
+          placementServiceOverride:
+            namespaces:
+              placement: /placement
+          messagePool:
+            size: 16384
+            watermark:
+              low: 0.2
+              high: 0.5
+  placementManager:
+    kvConfig:
+      namespace: /placement
+      environment: override_test_env
+      zone: embedded
+    placementWatcher:
+      key: m3aggregator
+      initWatchTimeout: 10s
+  hashType: murmur32
+  bufferDurationBeforeShardCutover: 10m
+  bufferDurationAfterShardCutoff: 10m
+  bufferDurationForFutureTimedMetric: 10m # Allow test to write into future.
+  bufferDurationForPastTimedMetric: 10s # Don't wait too long for timed metrics to flush.
+  resignTimeout: 1m
+  flushTimesManager:
+    kvConfig:
+      environment: override_test_env
+      zone: embedded
+    flushTimesKeyFmt: shardset/%d/flush
+    flushTimesPersistRetrier:
+      initialBackoff: 100ms
+      backoffFactor: 2.0
+      maxBackoff: 2s
+      maxRetries: 3
+  electionManager:
+    election:
+      leaderTimeout: 10s
+      resignTimeout: 10s
+      ttlSeconds: 10
+    serviceID:
+      name: m3aggregator
+      environment: override_test_env
+      zone: embedded
+    electionKeyFmt: shardset/%d/lock
+    campaignRetrier:
+      initialBackoff: 100ms
+      backoffFactor: 2.0
+      maxBackoff: 2s
+      forever: true
+      jitter: true
+    changeRetrier:
+      initialBackoff: 100ms
+      backoffFactor: 2.0
+      maxBackoff: 5s
+      forever: true
+      jitter: true
+    resignRetrier:
+      initialBackoff: 100ms
+      backoffFactor: 2.0
+      maxBackoff: 5s
+      forever: true
+      jitter: true
+    campaignStateCheckInterval: 1s
+    shardCutoffCheckOffset: 30s
+  flushManager:
+    checkEvery: 1s
+    jitterEnabled: true
+    maxJitters:
+      - flushInterval: 5s
+        maxJitterPercent: 1.0
+      - flushInterval: 10s
+        maxJitterPercent: 0.5
+      - flushInterval: 1m
+        maxJitterPercent: 0.5
+      - flushInterval: 10m
+        maxJitterPercent: 0.5
+      - flushInterval: 1h
+        maxJitterPercent: 0.25
+    numWorkersPerCPU: 0.5
+    flushTimesPersistEvery: 10s
+    maxBufferSize: 10m
+    forcedFlushWindowSize: 10s
+  flush:
+    handlers:
+      - dynamicBackend:
+          name: m3msg
+          hashType: murmur32
+          producer:
+            writer:
+              topicName: aggregated_metrics
+              topicServiceOverride:
+                zone: embedded
+                environment: override_test_env
+              messagePool:
+                size: 16384
+                watermark:
+                  low: 0.2
+                  high: 0.5
+  passthrough:
+    enabled: true
+  forwarding:
+    maxConstDelay: 1m # Need to add some buffer window, since timed metrics by default are delayed by 1min.
+  entryTTL: 1h
+  entryCheckInterval: 10m
+  maxTimerBatchSizePerWrite: 140
+  maxNumCachedSourceSets: 2
+  discardNaNAggregatedValues: true
+  entryPool:
+    size: 4096
+  counterElemPool:
+    size: 4096
+  timerElemPool:
+    size: 4096
+  gaugeElemPool:
+    size: 4096
diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml
@@ -0,0 +1,29 @@
+listenAddress: 0.0.0.0:7201
+
+logging:
+  level: info
+
+metrics:
+  scope:
+    prefix: "coordinator"
+  prometheus:
+    handlerPath: /metrics
+    listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved
+  sanitization: prometheus
+  samplingRate: 1.0
+  extended: none
+
+backend: noop-etcd
+clusterManagement:
+  etcd:
+    env: default_env
+    zone: embedded
+    service: m3db
+    cacheDir: /var/lib/m3kv
+    etcdClusters:
+    - zone: embedded
+      endpoints:
+      - etcd01:2379
+
+tagOptions:
+  idScheme: quoted