Merge branch 'r/mapping-rule-rollup-rule-config-support' of github.co…

…m:m3db/m3 into r/mapping-rule-rollup-rule-config-support

scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml

@@ -0,0 +1,28 @@
+version: "3.5"
+services:
+  dbnode01:
+    expose:
+      - "9000-9004"
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:9000-9004:9000-9004"
+      - "0.0.0.0:2379-2380:2379-2380"
+    networks:
+      - backend
+    image: "m3dbnode_integration:${REVISION}"
+  coordinator01:
+    expose:
+      - "7201"
+      - "7203"
+      - "7204"
+    ports:
+      - "0.0.0.0:7201:7201"
+      - "0.0.0.0:7203:7203"
+      - "0.0.0.0:7204:7204"
+    networks:
+      - backend
+    image: "m3coordinator_integration:${REVISION}"
+    volumes:
+      - "./:/etc/m3coordinator/"
+networks:
+  backend:

scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml

@@ -0,0 +1,64 @@
+listenAddress:
+  value: "0.0.0.0:7201"
+
+logging:
+  level: info
+
+metrics:
+  scope:
+    prefix: "coordinator"
+  prometheus:
+    handlerPath: /metrics
+    listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved
+  sanitization: prometheus
+  samplingRate: 1.0
+  extended: none
+
+clusters:
+  - namespaces:
+      - namespace: agg
+        type: aggregated
+        retention: 10h
+        resolution: 5s
+      - namespace: unagg
+        type: unaggregated
+        retention: 10m
+    client:
+      config:
+        service:
+          env: default_env
+          zone: embedded
+          service: m3db
+          cacheDir: /var/lib/m3kv
+          etcdClusters:
+            - zone: embedded
+              endpoints:
+                - dbnode01:2379
+      writeConsistencyLevel: majority
+      readConsistencyLevel: unstrict_majority
+
+downsample:
+  rules:
+    # mappingRules:
+    #   - filter: "foo:bar"
+    #     aggregation:
+    #       - 2
+    #     storagePolicies:
+    #       - retention: 12h
+    #         resolution: 10s
+    #     drop: false
+    rollupRules:
+      - filter: "foo:bar"
+        transforms:
+          - rollup:
+              metricName: "new_metric"
+            aggregate:
+              type: Sum
+        storagePolicies:
+          - retention: 10h
+            resolution: 5s
+        name: "testRollup"
+
+
+tagOptions:
+  idScheme: quoted

scripts/docker-integration-tests/coordinator_config_rules/test.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+
+set -xe
+
+source $GOPATH/src/github.com/m3db/m3/scripts/docker-integration-tests/common.sh
+REVISION=$(git rev-parse HEAD)
+SCRIPT_PATH=$GOPATH/src/github.com/m3db/m3/scripts/docker-integration-tests/coordinator_config_rules
+COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml
+EXPECTED_PATH=$SCRIPT_PATH/expected
+METRIC_NAME_TEST_OLD=foo
+export REVISION
+
+echo "Run m3dbnode and m3coordinator containers"
+docker-compose -f ${COMPOSE_FILE} up -d dbnode01
+docker-compose -f ${COMPOSE_FILE} up -d coordinator01
+
+# Think of this as a defer func() in golang
+function defer {
+  docker-compose -f ${COMPOSE_FILE} down || echo "unable to shutdown containers" # CI fails to stop all containers sometimes
+}
+trap defer EXIT
+
+setup_single_m3db_node
+
+function prometheus_remote_write {
+  local metric_name=$1
+  local datapoint_timestamp=$2
+  local datapoint_value=$3
+  local expect_success=$4
+  local expect_success_err=$5
+  local expect_status=$6
+  local expect_status_err=$7
+  local metrics_type=$8
+  local metrics_storage_policy=$9
+
+  network=$(docker network ls --format '{{.ID}}' | tail -n 1)
+  out=$((docker run -it --rm --network $network           \
+    $PROMREMOTECLI_IMAGE                                  \
+    -u http://coordinator01:7201/api/v1/prom/remote/write \
+    -t __name__:${metric_name}                            \
+    -t foo:bar
+    -h "M3-Metrics-Type: ${metrics_type}"                 \
+    -h "M3-Storage-Policy: ${metrics_storage_policy}"     \
+    -d ${datapoint_timestamp},${datapoint_value} | grep -v promremotecli_log) || true)
+  success=$(echo $out | grep -v promremotecli_log | docker run --rm -i $JQ_IMAGE jq .success)
+  status=$(echo $out | grep -v promremotecli_log | docker run --rm -i $JQ_IMAGE jq .statusCode)
+  if [[ "$success" != "$expect_success" ]]; then
+    echo $expect_success_err
+    return 1
+  fi
+  if [[ "$status" != "$expect_status" ]]; then
+    echo "${expect_status_err}: actual=${status}"
+    return 1
+  fi
+  echo "Returned success=${success}, status=${status} as expected"
+  return 0
+}
+
+function prometheus_write_metric {
+  echo "Test write with aggregated metrics type works as expected"
+  prometheus_remote_write \
+    old_metric now 84.84 \
+    true "Expected request to succeed" \
+    200 "Expected request to return status code 200" \
+    aggregated 15s:10h
+}
+
+function prometheus_query_native {
+  local endpoint=${endpoint:-}
+  local query=${query:-}
+  local params=${params:-}
+  local metrics_type=${metrics_type:-}
+  local metrics_storage_policy=${metrics_storage_policy:-}
+  local jq_path=${jq_path:-}
+  local expected_value=${expected_value:-}
+
+  params_prefixed=""
+  if [[ "$params" != "" ]]; then
+    params_prefixed='&'"${params}"
+  fi
+
+  result=$(curl -s                                    \
+    -H "M3-Metrics-Type: ${metrics_type}"             \
+    -H "M3-Storage-Policy: ${metrics_storage_policy}" \
+    "0.0.0.0:7201/api/v1/${endpoint}?query=${query}${params_prefixed}" | jq -r "${jq_path}")
+  test "$result" = "$expected_value"
+  return $?
+}
+
+
+function test_query_rollup_rule {
+  now=$(date +"%s")
+  hour_ago=$(expr $now - 3600) 
+  step="30s"
+  params_instant=""
+  params_range="start=${hour_ago}"'&'"end=${now}"'&'"step=30s"
+  jq_path_instant=".data.result[0].value[1]"
+  jq_path_range=".data.result[0].metric[\"__name__\"]"
+
+  echo "Test query rollup rule"
+  ATTEMPTS=50 TIMEOUT=2 MAX_TIMEOUT=4 \
+    endpoint=query_range query=new_metric params="$params_range" \
+    metrics_type="aggregated" metrics_storage_policy="15s:10h" jq_path="$jq_path_range" expected_value="new_metric" \
+    retry_with_backoff prometheus_query_native
+}
+
+echo "Running prometehus tests"
+test_query_rollup_rule

scripts/docker-integration-tests/run.sh

@@ -3,17 +3,18 @@
 set -ex
 
 TESTS=(
-	scripts/docker-integration-tests/simple/test.sh
-	scripts/docker-integration-tests/cold_writes_simple/test.sh
-	scripts/docker-integration-tests/prometheus/test.sh
-	scripts/docker-integration-tests/prometheus_replication/test.sh
-	scripts/docker-integration-tests/carbon/test.sh
-	scripts/docker-integration-tests/aggregator/test.sh
-	scripts/docker-integration-tests/query_fanout/test.sh
-	scripts/docker-integration-tests/repair/test.sh
-	scripts/docker-integration-tests/replication/test.sh
-	scripts/docker-integration-tests/repair_and_replication/test.sh
-	scripts/docker-integration-tests/multi_cluster_write/test.sh
+	# scripts/docker-integration-tests/simple/test.sh
+	# scripts/docker-integration-tests/cold_writes_simple/test.sh
+	# scripts/docker-integration-tests/prometheus/test.sh
+	# scripts/docker-integration-tests/prometheus_replication/test.sh
+	# scripts/docker-integration-tests/carbon/test.sh
+	# scripts/docker-integration-tests/aggregator/test.sh
+	# scripts/docker-integration-tests/query_fanout/test.sh
+	# scripts/docker-integration-tests/repair/test.sh
+	# scripts/docker-integration-tests/replication/test.sh
+	# scripts/docker-integration-tests/repair_and_replication/test.sh
+	# scripts/docker-integration-tests/multi_cluster_write/test.sh
+	scripts/docker-integration-tests/coordinator_config_rules/test.sh
 )
 
 scripts/docker-integration-tests/setup.sh

src/cmd/services/m3query/config/config.go

@@ -134,6 +134,9 @@ type Configuration struct {
 	// ResultOptions are the results options for query.
 	ResultOptions ResultOptions `yaml:"resultOptions"`
 
+	// Experimental is the configuration for the experimental API group.
+	Experimental ExperimentalAPIConfiguration `yaml:"experimental"`
+
 	// Cache configurations.
 	//
 	// Deprecated: cache configurations are no longer supported. Remove from file
@@ -512,3 +515,8 @@ func TagOptionsFromConfig(cfg TagOptionsConfiguration) (models.TagOptions, error
 
 	return opts, nil
 }
+
+// ExperimentalAPIConfiguration is the configuration for the experimental API group.
+type ExperimentalAPIConfiguration struct {
+	Enabled bool `yaml:"enabled"`
+}