From 8720c2b51c1c93b46d402b2b8c3d73d2848e0e8c Mon Sep 17 00:00:00 2001 From: jiangli Date: Thu, 7 Mar 2019 16:54:34 +0800 Subject: [PATCH 1/6] add half-open and fix degrade-bug --- sentinel-core/pom.xml | 6 - .../com/alibaba/csp/sentinel/node/Node.java | 38 ++++- .../csp/sentinel/node/StatisticNode.java | 71 +++++++++- .../sentinel/slots/block/RuleConstant.java | 9 +- .../slots/block/degrade/DegradeRule.java | 133 ++++++++++++++---- .../slots/statistic/base/LongAdder.java | 6 +- .../slots/statistic/data/MetricBucket.java | 22 +++ .../slots/statistic/metric/ArrayMetric.java | 35 +++++ .../slots/statistic/metric/Metric.java | 22 +++ .../slots/block/degrade/DegradeTest.java | 42 ++++-- 10 files changed, 328 insertions(+), 56 deletions(-) diff --git a/sentinel-core/pom.xml b/sentinel-core/pom.xml index 5b14f78f64..bb842b3e2b 100755 --- a/sentinel-core/pom.xml +++ b/sentinel-core/pom.xml @@ -16,7 +16,6 @@ junit junit - test org.hamcrest @@ -31,27 +30,22 @@ org.mockito mockito-core - test org.awaitility awaitility - test org.hamcrest java-hamcrest - test org.powermock powermock-module-junit4 - test org.powermock powermock-api-mockito2 - test diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/Node.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/Node.java index b212c782f6..b760f2f698 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/Node.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/Node.java @@ -16,6 +16,9 @@ package com.alibaba.csp.sentinel.node; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import com.alibaba.csp.sentinel.Entry; import com.alibaba.csp.sentinel.node.metric.MetricNode; @@ -126,10 +129,22 @@ public interface Node { */ void addPassRequest(int count); + /** + * minus exception + */ + void minusMinuteException(); + + void minusSecondException(); + + /** + * minus Rt + */ + void minusRt(int count); + /** * Add rt and success count. * - * @param rt response time + * @param rt response time * @param success success count to add */ void addRtAndSuccess(long rt, int success); @@ -164,4 +179,25 @@ public interface Node { * Debug only. */ void debug(); + + /** + * reset lastRT + */ + void resetLastRt(); + + /** + * get lastRT + */ + AtomicLong getLastRt(); + + /** + * get last Result (true:success;false:exception) + */ + AtomicBoolean getLastResult(); + + /** + * get last rt(same TimeWindow) sum + */ + AtomicInteger getLastRtSum(); + } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/StatisticNode.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/StatisticNode.java index a27ef0fe52..d2ce0e384f 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/StatisticNode.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/StatisticNode.java @@ -18,7 +18,9 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import com.alibaba.csp.sentinel.util.TimeUtil; import com.alibaba.csp.sentinel.node.metric.MetricNode; @@ -93,7 +95,7 @@ public class StatisticNode implements Node { * by given {@code sampleCount}. */ private transient volatile Metric rollingCounterInSecond = new ArrayMetric(SampleCountProperty.SAMPLE_COUNT, - IntervalProperty.INTERVAL); + IntervalProperty.INTERVAL); /** * Holds statistics of the recent 60 seconds. The windowLengthInMs is deliberately set to 1000 milliseconds, @@ -110,6 +112,21 @@ public class StatisticNode implements Node { * The last timestamp when metrics were fetched. */ private long lastFetchTime = -1; + /** + * record last rt(Prepare degrade rt sum and same TimeWindow) + */ + private AtomicLong lastRt = new AtomicLong(0); + + /** + * record last rt(same TimeWindow) sum + */ + private AtomicInteger lastRtSum = new AtomicInteger(0); + /** + * lastResult + * + * @return + */ + private AtomicBoolean lastResult = new AtomicBoolean(true); @Override public Map metrics() { @@ -137,7 +154,7 @@ private boolean isNodeInTime(MetricNode node, long currentTime) { private boolean isValidMetricNode(MetricNode node) { return node.getPassQps() > 0 || node.getBlockQps() > 0 || node.getSuccessQps() > 0 - || node.getExceptionQps() > 0 || node.getRt() > 0; + || node.getExceptionQps() > 0 || node.getRt() > 0; } @Override @@ -232,11 +249,37 @@ public void addPassRequest(int count) { rollingCounterInMinute.addPass(count); } + @Override + public void minusSecondException() { + rollingCounterInSecond.minusException(); + } + + @Override + public void minusMinuteException() { + rollingCounterInMinute.minusException(); + } + + @Override + public void minusRt(int count) { + rollingCounterInSecond.minusRt(lastRt.get()); + rollingCounterInSecond.minusSuccess(count); + rollingCounterInMinute.minusRt(lastRt.get()); + rollingCounterInMinute.minusSuccess(count); + lastRt.set(0); + } + @Override public void addRtAndSuccess(long rt, int successCount) { rollingCounterInSecond.addSuccess(successCount); rollingCounterInSecond.addRT(rt); - + if (rollingCounterInSecond.newTimeWindow()) { + lastRt.set(rt); + lastRtSum.set(1); + } else { + lastRt.set(rt + lastRt.get()); + lastRtSum.getAndAdd(1); + } + lastResult.set(true); rollingCounterInMinute.addSuccess(successCount); rollingCounterInMinute.addRT(rt); } @@ -251,7 +294,13 @@ public void increaseBlockQps(int count) { public void increaseExceptionQps(int count) { rollingCounterInSecond.addException(count); rollingCounterInMinute.addException(count); + lastResult.set(false); + + } + @Override + public void resetLastRt() { + lastRt.set(0); } @Override @@ -268,4 +317,20 @@ public void decreaseThreadNum() { public void debug() { rollingCounterInSecond.debugQps(); } + + @Override + public AtomicLong getLastRt() { + return lastRt; + } + + @Override + public AtomicBoolean getLastResult() { + return lastResult; + } + + @Override + public AtomicInteger getLastRtSum() { + return lastRtSum; + } + } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/RuleConstant.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/RuleConstant.java index 978b169fbd..f5eb0cbab1 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/RuleConstant.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/RuleConstant.java @@ -50,9 +50,16 @@ public final class RuleConstant { public static final String LIMIT_APP_DEFAULT = "default"; public static final String LIMIT_APP_OTHER = "other"; + /** + * Three states of the degraded switch + */ + public static final int DEGRADE_CUT_CLOSE = 0; + public static final int DEGRADE_CUT_OPEN = 1; + public static final int DEGRADE_CUT_HALF_OPEN = 2; public static final int DEFAULT_SAMPLE_COUNT = 2; public static final int DEFAULT_WINDOW_INTERVAL_MS = 1000; - private RuleConstant() {} + private RuleConstant() { + } } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRule.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRule.java index 8abb1ddb83..78c65b85d8 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRule.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRule.java @@ -58,9 +58,10 @@ public class DegradeRule extends AbstractRule { private static final int RT_MAX_EXCEED_N = 5; private static ScheduledExecutorService pool = Executors.newScheduledThreadPool( - Runtime.getRuntime().availableProcessors(), new NamedThreadFactory("sentinel-degrade-reset-task", true)); + Runtime.getRuntime().availableProcessors(), new NamedThreadFactory("sentinel-degrade-reset-task", true)); - public DegradeRule() {} + public DegradeRule() { + } public DegradeRule(String resourceName) { setResource(resourceName); @@ -81,7 +82,16 @@ public DegradeRule(String resourceName) { */ private int grade = RuleConstant.DEGRADE_GRADE_RT; - private final AtomicBoolean cut = new AtomicBoolean(false); + /** + * First request after degrade + */ + private AtomicBoolean firstRequest = new AtomicBoolean(true); + + private volatile int cut = RuleConstant.DEGRADE_CUT_CLOSE; + /** + * Degrade half-open-switch(default=false) + */ + private boolean enableHalfOpen = false; public int getGrade() { return grade; @@ -94,6 +104,8 @@ public DegradeRule setGrade(int grade) { private AtomicLong passCount = new AtomicLong(0); + private final Object lock = new Object(); + public double getCount() { return count; } @@ -103,14 +115,23 @@ public DegradeRule setCount(double count) { return this; } - private boolean isCut() { - return cut.get(); + public int getCut() { + return cut; + } + + public void setCut(int cut) { + this.cut = cut; + } + + public boolean isEnableHalfOpen() { + return enableHalfOpen; } - private void setCut(boolean cut) { - this.cut.set(cut); + public void setEnableHalfOpen(boolean enableHalfOpen) { + this.enableHalfOpen = enableHalfOpen; } + public AtomicLong getPassCount() { return passCount; } @@ -136,7 +157,7 @@ public boolean equals(Object o) { return false; } - DegradeRule that = (DegradeRule)o; + DegradeRule that = (DegradeRule) o; if (count != that.count) { return false; @@ -161,27 +182,65 @@ public int hashCode() { @Override public boolean passCheck(Context context, DefaultNode node, int acquireCount, Object... args) { - if (cut.get()) { + if (cut == RuleConstant.DEGRADE_CUT_OPEN) { return false; } - ClusterNode clusterNode = ClusterBuilderSlot.getClusterNode(this.getResource()); if (clusterNode == null) { return true; } - + // degrade_cut_half_open + if (cut == RuleConstant.DEGRADE_CUT_HALF_OPEN) { + //First request after downgrade + if (firstRequest.get()) { + firstRequest.set(false); + return true; + } + //In the half-open state, only five requests are all normal and will be fully opened. + if (grade == RuleConstant.DEGRADE_GRADE_RT) { + long rt = clusterNode.getLastRt().get(); + if (rt == 0) { + return degradeCutOpen(); + } + if (rt < this.count) { + cut = RuleConstant.DEGRADE_CUT_CLOSE; + return true; + } + clusterNode.minusRt(1); + return degradeCutOpen(); + } + if (grade != RuleConstant.DEGRADE_GRADE_RT) { + //When the sliding window slides over the next window, it needs to be cleared. + if (clusterNode.getLastResult().get()) { + cut = RuleConstant.DEGRADE_CUT_CLOSE; + return true; + } + if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO) { + clusterNode.minusSecondException(); + } else { + clusterNode.minusMinuteException(); + } + return degradeCutOpen(); + } + } + // degrade_cut_close if (grade == RuleConstant.DEGRADE_GRADE_RT) { double rt = clusterNode.avgRt(); if (rt < this.count) { passCount.set(0); + clusterNode.resetLastRt(); return true; } - // Sentinel will degrade the service only if count exceeds. if (passCount.incrementAndGet() < RT_MAX_EXCEED_N) { return true; } - } else if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO) { + clusterNode.minusRt(clusterNode.getLastRtSum().get()); + clusterNode.resetLastRt(); + passCount.set(0); + return degradeCutOpen(); + } + if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO) { double exception = clusterNode.exceptionQps(); double success = clusterNode.successQps(); long total = clusterNode.totalQps(); @@ -198,30 +257,29 @@ public boolean passCheck(Context context, DefaultNode node, int acquireCount, Ob if (exception / success < count) { return true; } - } else if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_COUNT) { + clusterNode.minusSecondException(); + return degradeCutOpen(); + } + if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_COUNT) { double exception = clusterNode.totalException(); if (exception < count) { return true; } + // after degrade-open minus exceptionCount + clusterNode.minusMinuteException(); } - - if (cut.compareAndSet(false, true)) { - ResetTask resetTask = new ResetTask(this); - pool.schedule(resetTask, timeWindow, TimeUnit.SECONDS); - } - - return false; + return degradeCutOpen(); } @Override public String toString() { return "DegradeRule{" + - "resource=" + getResource() + - ", grade=" + grade + - ", count=" + count + - ", limitApp=" + getLimitApp() + - ", timeWindow=" + timeWindow + - "}"; + "resource=" + getResource() + + ", grade=" + grade + + ", count=" + count + + ", limitApp=" + getLimitApp() + + ", timeWindow=" + timeWindow + + "}"; } private static final class ResetTask implements Runnable { @@ -234,9 +292,26 @@ private static final class ResetTask implements Runnable { @Override public void run() { - rule.getPassCount().set(0); - rule.cut.set(false); + //enableHalfOpen + if (rule.enableHalfOpen) { + rule.setCut(RuleConstant.DEGRADE_CUT_HALF_OPEN); + } else { + rule.setCut(RuleConstant.DEGRADE_CUT_CLOSE); + } + } + } + + private boolean degradeCutOpen() { + synchronized (lock) { + if (cut != RuleConstant.DEGRADE_CUT_OPEN) { + // Automatically degrade. + cut = RuleConstant.DEGRADE_CUT_OPEN; + ResetTask resetTask = new ResetTask(this); + pool.schedule(resetTask, timeWindow, TimeUnit.SECONDS); + } + return false; } } + } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/base/LongAdder.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/base/LongAdder.java index beeaedcf67..671f1ad655 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/base/LongAdder.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/base/LongAdder.java @@ -82,10 +82,10 @@ public void increment() { } /** - * Equivalent to {@code add(-1)}. + * Equivalent to {@code add(-x)}. */ - public void decrement() { - add(-1L); + public void decrement(long x) { + add(-x); } /** diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/data/MetricBucket.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/data/MetricBucket.java index ab1dc79634..5077ca95e2 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/data/MetricBucket.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/data/MetricBucket.java @@ -57,6 +57,11 @@ public MetricBucket reset() { return this; } + public MetricBucket reset(MetricEvent event) { + counters[event.ordinal()].reset(); + return this; + } + public long get(MetricEvent event) { return counters[event.ordinal()].sum(); } @@ -66,6 +71,11 @@ public MetricBucket add(MetricEvent event, long n) { return this; } + public MetricBucket minus(MetricEvent event, long n) { + counters[event.ordinal()].decrement(n); + return this; + } + public long pass() { return get(MetricEvent.PASS); } @@ -98,6 +108,18 @@ public void addException(int n) { add(MetricEvent.EXCEPTION, n); } + public void minusException(int n) { + minus(MetricEvent.EXCEPTION, n); + } + + public void minusRt(long rt) { + minus(MetricEvent.RT, rt); + + } + + public void minusSuccess(int n) { + minus(MetricEvent.SUCCESS, n); + } public void addBlock(int n) { add(MetricEvent.BLOCK, n); } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/ArrayMetric.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/ArrayMetric.java index cadec6ea58..12b516a029 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/ArrayMetric.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/ArrayMetric.java @@ -191,6 +191,33 @@ public void addRT(long rt) { wrap.value().addRT(rt); } + @Override + public void minusSuccess(int n) { + if (data.values().isEmpty()) { + return; + } + WindowWrap wrap = data.currentWindow(); + wrap.value().minusSuccess(n); + } + + @Override + public void minusRt(long rt) { + if (data.values().isEmpty()) { + return; + } + WindowWrap wrap = data.currentWindow(); + wrap.value().minusRt(rt); + } + + @Override + public void minusException() { + if (data.values().isEmpty()) { + return; + } + WindowWrap wrap = data.currentWindow(); + wrap.value().minusException(1); + } + @Override public void debugQps() { data.currentWindow(); @@ -235,4 +262,12 @@ public double getWindowIntervalInSec() { public int getSampleCount() { return data.getSampleCount(); } + + @Override + public boolean newTimeWindow() { + if (data.values().isEmpty()) { + return true; + } + return false; + } } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/Metric.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/Metric.java index a3fb933bf6..9c6a527cd8 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/Metric.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/Metric.java @@ -118,6 +118,28 @@ public interface Metric { */ void addRT(long rt); + /** + * is New TimeWindow + */ + boolean newTimeWindow(); + + /** + * minus success count + */ + void minusSuccess(int n); + + /** + * minus rt + */ + void minusRt(long rt); + + /** + * minus exception + * + * @return + */ + void minusException(); + double getWindowIntervalInSec(); int getSampleCount(); diff --git a/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java index faf78931d6..647a10f542 100755 --- a/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java +++ b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java @@ -20,6 +20,7 @@ import static org.mockito.Mockito.when; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.Test; @@ -41,12 +42,11 @@ public void testAverageRtDegrade() throws InterruptedException { String key = "test_degrade_average_rt"; ClusterNode cn = mock(ClusterNode.class); ClusterBuilderSlot.getClusterNodeMap().put(new StringResourceWrapper(key, EntryType.IN), cn); - Context context = mock(Context.class); DefaultNode node = mock(DefaultNode.class); when(node.getClusterNode()).thenReturn(cn); when(cn.avgRt()).thenReturn(2L); - + when(cn.getLastRtSum()).thenReturn(new AtomicInteger(0)); DegradeRule rule = new DegradeRule(); rule.setCount(1); rule.setResource(key); @@ -59,10 +59,14 @@ public void testAverageRtDegrade() throws InterruptedException { // The third time will fail. assertFalse(rule.passCheck(context, node, 1)); assertFalse(rule.passCheck(context, node, 1)); - // Restore. - TimeUnit.MILLISECONDS.sleep(2200); - assertTrue(rule.passCheck(context, node, 1)); + + TimeUnit.SECONDS.sleep(6); + rule.setCount(3); + //When the fifth request ends, degrade will blow close. + for (int i = 0; i < 10; i++) { + assertTrue(rule.passCheck(context, node, 1)); + } } @Test @@ -92,9 +96,21 @@ public void testExceptionRatioModeDegrade() throws Throwable { // Restore from the degrade timeout. TimeUnit.MILLISECONDS.sleep(2200); - when(cn.successQps()).thenReturn(20L); + // when(cn.successQps()).thenReturn(20L); // Will pass. - assertTrue(rule.passCheck(context, node, 1)); + when(cn.exceptionQps()).thenReturn(0L); + for (int i = 0; i < 5; i++) { + assertTrue(rule.passCheck(context, node, 1)); + } + when(cn.exceptionQps()).thenReturn(20L); + assertFalse(rule.passCheck(context, node, 1)); + TimeUnit.SECONDS.sleep(6); + // Will pass. + //When the fifth request ends, degrade will blow close. + when(cn.exceptionQps()).thenReturn(0L); + for (int i = 0; i < 5; i++) { + assertTrue(rule.passCheck(context, node, 1)); + } } @Test @@ -115,16 +131,16 @@ public void testExceptionCountModeDegrade() throws Throwable { rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION_COUNT); when(cn.totalException()).thenReturn(4L); - // Will fail. assertFalse(rule.passCheck(context, node, 1)); - // Restore from the degrade timeout. - TimeUnit.MILLISECONDS.sleep(2200); - when(cn.totalException()).thenReturn(0L); - // Will pass. - assertTrue(rule.passCheck(context, node, 1)); + TimeUnit.SECONDS.sleep(3); + when(cn.totalException()).thenReturn(3L); + //When the fifth request ends, degrade will blow close. + for (int i = 0; i < 10; i++) { + assertTrue(rule.passCheck(context, node, 1)); + } } } From 3a8f2d1336a3bcb682b937d26ea7c2c292d70518 Mon Sep 17 00:00:00 2001 From: jiangli Date: Thu, 14 Mar 2019 09:52:15 +0800 Subject: [PATCH 2/6] merge into master --- .../csp/sentinel/slots/block/degrade/DegradeTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java index 0a06437788..38eaaf5d71 100755 --- a/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java +++ b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java @@ -103,16 +103,16 @@ public void testExceptionRatioModeDegrade() throws Throwable { when(cn.successQps()).thenReturn(20d); // Will pass. - when(cn.exceptionQps()).thenReturn(0L); + when(cn.exceptionQps()).thenReturn(0d); for (int i = 0; i < 5; i++) { assertTrue(rule.passCheck(context, node, 1)); } - when(cn.exceptionQps()).thenReturn(20L); + when(cn.exceptionQps()).thenReturn(20d); assertFalse(rule.passCheck(context, node, 1)); TimeUnit.SECONDS.sleep(6); // Will pass. //When the fifth request ends, degrade will blow close. - when(cn.exceptionQps()).thenReturn(0L); + when(cn.exceptionQps()).thenReturn(0d); for (int i = 0; i < 5; i++) { assertTrue(rule.passCheck(context, node, 1)); } From 343eb3d78231bbeb7a2772bcce63d7ccb62d36ff Mon Sep 17 00:00:00 2001 From: jiangli Date: Thu, 14 Mar 2019 10:48:59 +0800 Subject: [PATCH 3/6] merge into master --- .../slots/statistic/metric/ArrayMetric.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/ArrayMetric.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/ArrayMetric.java index 53549ed7ca..62f6f03097 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/ArrayMetric.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/ArrayMetric.java @@ -254,20 +254,6 @@ public void minusException() { wrap.value().minusException(1); } - @Override - public void debugQps() { - data.currentWindow(); - StringBuilder sb = new StringBuilder(); - sb.append(Thread.currentThread().getId()).append("_"); - for (WindowWrap windowWrap : data.list()) { - - sb.append(windowWrap.windowStart()).append(":").append(windowWrap.value().pass()).append(":") - .append(windowWrap.value().block()); - sb.append(","); - - } - System.out.println(sb); - public void debug() { data.debug(System.currentTimeMillis()); } From e39cf0737a051cf5063b7f26fa067ce1e6cbef6f Mon Sep 17 00:00:00 2001 From: jiangli Date: Thu, 14 Mar 2019 11:02:04 +0800 Subject: [PATCH 4/6] =?UTF-8?q?=E3=80=82=E3=80=82=E8=BF=B7=E4=B9=8Bci?= =?UTF-8?q?=E6=9E=84=E5=BB=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/alibaba/csp/sentinel/slots/statistic/metric/Metric.java | 1 + 1 file changed, 1 insertion(+) diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/Metric.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/Metric.java index 2ef999eed5..7c5ef2da04 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/Metric.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/Metric.java @@ -148,6 +148,7 @@ public interface Metric extends DebugSupport { */ void minusException(); + /** * Get the sliding window length in seconds. * * @return the sliding window length From 44aa7871431508dbc6387ae4b68316208abbe637 Mon Sep 17 00:00:00 2001 From: jiangli Date: Thu, 21 Mar 2019 17:12:29 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E9=99=8D=E7=BA=A7=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=8D=95=E7=8B=AC=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/alibaba/csp/sentinel/node/Node.java | 41 +++---- .../csp/sentinel/node/StatisticNode.java | 101 +++++++++--------- .../slots/block/degrade/DegradeRule.java | 45 ++++---- .../slots/block/degrade/DegradeTest.java | 11 +- 4 files changed, 101 insertions(+), 97 deletions(-) diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/Node.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/Node.java index fe33df8498..6eacb60a91 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/Node.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/Node.java @@ -16,10 +16,6 @@ package com.alibaba.csp.sentinel.node; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - import com.alibaba.csp.sentinel.Entry; import com.alibaba.csp.sentinel.node.metric.MetricNode; import com.alibaba.csp.sentinel.slots.statistic.metric.DebugSupport; @@ -156,18 +152,6 @@ public interface Node extends OccupySupport, DebugSupport { */ void addPassRequest(int count); - /** - * minus exception - */ - void minusMinuteException(); - - void minusSecondException(); - - /** - * minus Rt - */ - void minusRt(int count); - /** * Add rt and success count. * @@ -212,23 +196,32 @@ public interface Node extends OccupySupport, DebugSupport { void debug(); /** - * reset lastRT + * Get the average RT in the degraded state + */ + double getDegradeAvgRt(); + + /** + * Get the total number of exceptions for the current minute */ - void resetLastRt(); + long getDegradeMinusExceptionCount(); /** - * get lastRT + * Get the abnormal proportion in the degraded state */ - AtomicLong getLastRt(); + double getDegradeSecondExceptionRatio(); /** - * get last Result (true:success;false:exception) + * reset rt */ - AtomicBoolean getLastResult(); + void resetDegradeRt(); /** - * get last rt(same TimeWindow) sum + * reset exception-ratio */ - AtomicInteger getLastRtSum(); + void resetDegradeExceptionDataRatio(); + /** + * reset exception-count + */ + void resetDegradeExceptionCount(); } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/StatisticNode.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/StatisticNode.java index 80484cebce..9339e0277e 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/StatisticNode.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/StatisticNode.java @@ -18,10 +18,9 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; +import com.alibaba.csp.sentinel.slots.statistic.base.LongAdder; import com.alibaba.csp.sentinel.util.TimeUtil; import com.alibaba.csp.sentinel.node.metric.MetricNode; import com.alibaba.csp.sentinel.slots.statistic.metric.ArrayMetric; @@ -112,21 +111,31 @@ public class StatisticNode implements Node { * The last timestamp when metrics were fetched. */ private long lastFetchTime = -1; + + /** + * Record the number of exceptions in the next second + */ + private LongAdder degradeSecondExceptionCount = new LongAdder(); + /** - * record last rt(Prepare degrade rt sum and same TimeWindow) + * Record the number of traffic passes in one second */ - private AtomicLong lastRt = new AtomicLong(0); + private LongAdder degradeSecondPassCount = new LongAdder(); /** - * record last rt(same TimeWindow) sum + * Record the number of exceptions in one minute */ - private AtomicInteger lastRtSum = new AtomicInteger(0); + private LongAdder degradeMinusExceptionCount = new LongAdder(); + + /** + * Record the number of successful traffic in one second + */ + private LongAdder degradeSecondSuccessCount = new LongAdder(); + /** - * lastResult - * - * @return + * Record one second total RT */ - private AtomicBoolean lastResult = new AtomicBoolean(true); + private LongAdder degradeSecondTotalRT = new LongAdder(); @Override public Map metrics() { @@ -154,7 +163,7 @@ private boolean isNodeInTime(MetricNode node, long currentTime) { private boolean isValidMetricNode(MetricNode node) { return node.getPassQps() > 0 || node.getBlockQps() > 0 || node.getSuccessQps() > 0 - || node.getExceptionQps() > 0 || node.getRt() > 0 || node.getOccupiedPassQps() > 0; + || node.getExceptionQps() > 0 || node.getRt() > 0 || node.getOccupiedPassQps() > 0; } @Override @@ -257,25 +266,10 @@ public int curThreadNum() { public void addPassRequest(int count) { rollingCounterInSecond.addPass(count); rollingCounterInMinute.addPass(count); - } - - @Override - public void minusSecondException() { - rollingCounterInSecond.minusException(); - } - - @Override - public void minusMinuteException() { - rollingCounterInMinute.minusException(); - } - - @Override - public void minusRt(int count) { - rollingCounterInSecond.minusRt(lastRt.get()); - rollingCounterInSecond.minusSuccess(count); - rollingCounterInMinute.minusRt(lastRt.get()); - rollingCounterInMinute.minusSuccess(count); - lastRt.set(0); + if (rollingCounterInSecond.newTimeWindow()) { + degradeSecondPassCount.reset(); + } + degradeSecondPassCount.add(count); } @Override @@ -283,13 +277,11 @@ public void addRtAndSuccess(long rt, int successCount) { rollingCounterInSecond.addSuccess(successCount); rollingCounterInSecond.addRT(rt); if (rollingCounterInSecond.newTimeWindow()) { - lastRt.set(rt); - lastRtSum.set(1); - } else { - lastRt.set(rt + lastRt.get()); - lastRtSum.getAndAdd(1); + degradeSecondSuccessCount.reset(); + degradeSecondTotalRT.reset(); } - lastResult.set(true); + degradeSecondSuccessCount.add(successCount); + degradeSecondTotalRT.add(rt); rollingCounterInMinute.addSuccess(successCount); rollingCounterInMinute.addRT(rt); } @@ -304,12 +296,6 @@ public void increaseBlockQps(int count) { public void increaseExceptionQps(int count) { rollingCounterInSecond.addException(count); rollingCounterInMinute.addException(count); - lastResult.set(false); - } - - @Override - public void resetLastRt() { - lastRt.set(0); } @Override @@ -379,18 +365,37 @@ public void addOccupiedPass(int acquireCount) { } @Override - public AtomicLong getLastRt() { - return lastRt; + public double getDegradeAvgRt() { + if (degradeSecondExceptionCount.longValue() == 0) { + return 0; + } + return degradeSecondTotalRT.doubleValue() * 1.0 / degradeSecondSuccessCount.doubleValue(); + } + + @Override + public long getDegradeMinusExceptionCount() { + return degradeMinusExceptionCount.longValue(); + } + + @Override + public double getDegradeSecondExceptionRatio() { + return degradeSecondExceptionCount.doubleValue() / degradeSecondPassCount.doubleValue(); } @Override - public AtomicBoolean getLastResult() { - return lastResult; + public void resetDegradeRt() { + degradeSecondTotalRT.reset(); + degradeSecondSuccessCount.reset(); } @Override - public AtomicInteger getLastRtSum() { - return lastRtSum; + public void resetDegradeExceptionDataRatio() { + degradeSecondExceptionCount.reset(); + degradeSecondPassCount.reset(); } + @Override + public void resetDegradeExceptionCount() { + degradeMinusExceptionCount.reset(); + } } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRule.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRule.java index eb29cbeb14..2a2956823b 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRule.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRule.java @@ -198,45 +198,50 @@ public boolean passCheck(Context context, DefaultNode node, int acquireCount, Ob } //In the half-open state, only five requests are all normal and will be fully opened. if (grade == RuleConstant.DEGRADE_GRADE_RT) { - long rt = clusterNode.getLastRt().get(); - if (rt == 0) { - return degradeCutOpen(); - } + + double rt = clusterNode.getDegradeAvgRt(); + if (rt < this.count) { cut = RuleConstant.DEGRADE_CUT_CLOSE; return true; } - clusterNode.minusRt(1); + clusterNode.resetDegradeRt(); return degradeCutOpen(); } - if (grade != RuleConstant.DEGRADE_GRADE_RT) { - //When the sliding window slides over the next window, it needs to be cleared. - if (clusterNode.getLastResult().get()) { + + if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO) { + if (clusterNode.getDegradeSecondExceptionRatio() < this.count) { cut = RuleConstant.DEGRADE_CUT_CLOSE; return true; } - if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO) { - clusterNode.minusSecondException(); - } else { - clusterNode.minusMinuteException(); + //此处需要reset + clusterNode.resetDegradeExceptionDataRatio(); + return degradeCutOpen(); + } + + if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_COUNT) { + + if (clusterNode.getDegradeMinusExceptionCount() == 0) { + cut = RuleConstant.DEGRADE_CUT_CLOSE; + return true; } + //When the sliding window slides over the next window, it needs to be cleared. + clusterNode.resetDegradeExceptionCount(); return degradeCutOpen(); } } // degrade_cut_close if (grade == RuleConstant.DEGRADE_GRADE_RT) { - double rt = clusterNode.avgRt(); + double rt = clusterNode.getDegradeAvgRt(); if (rt < this.count) { passCount.set(0); - clusterNode.resetLastRt(); return true; } // Sentinel will degrade the service only if count exceeds. if (passCount.incrementAndGet() < RT_MAX_EXCEED_N) { return true; } - clusterNode.minusRt(clusterNode.getLastRtSum().get()); - clusterNode.resetLastRt(); + clusterNode.resetDegradeRt(); passCount.set(0); return degradeCutOpen(); } @@ -257,16 +262,16 @@ public boolean passCheck(Context context, DefaultNode node, int acquireCount, Ob if (exception / success < count) { return true; } - clusterNode.minusSecondException(); + clusterNode.resetDegradeExceptionDataRatio(); return degradeCutOpen(); } if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_COUNT) { - double exception = clusterNode.totalException(); + double exception = clusterNode.getDegradeMinusExceptionCount(); if (exception < count) { return true; } - // after degrade-open minus exceptionCount - clusterNode.minusMinuteException(); + clusterNode.resetDegradeExceptionCount(); + } return degradeCutOpen(); } diff --git a/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java index 38eaaf5d71..cd32eec6a3 100755 --- a/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java +++ b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java @@ -46,8 +46,7 @@ public void testAverageRtDegrade() throws InterruptedException { DefaultNode node = mock(DefaultNode.class); when(node.getClusterNode()).thenReturn(cn); - when(cn.getLastRtSum()).thenReturn(new AtomicInteger(0)); - when(cn.avgRt()).thenReturn(2d); + when(cn.getDegradeAvgRt()).thenReturn(2d); DegradeRule rule = new DegradeRule(); rule.setCount(1); @@ -76,6 +75,7 @@ public void testExceptionRatioModeDegrade() throws Throwable { String key = "test_degrade_exception_ratio"; ClusterNode cn = mock(ClusterNode.class); when(cn.exceptionQps()).thenReturn(2d); + when(cn.getDegradeSecondExceptionRatio()).thenReturn(0.2d); // Indicates that there are QPS more than min threshold. when(cn.totalQps()).thenReturn(12d); ClusterBuilderSlot.getClusterNodeMap().put(new StringResourceWrapper(key, EntryType.IN), cn); @@ -91,6 +91,7 @@ public void testExceptionRatioModeDegrade() throws Throwable { rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO); when(cn.successQps()).thenReturn(8d); + // when(cn.getDegradeSecondExceptionRatio()).thenReturn() // Will fail. assertFalse(rule.passCheck(context, node, 1)); @@ -122,7 +123,7 @@ public void testExceptionRatioModeDegrade() throws Throwable { public void testExceptionCountModeDegrade() throws Throwable { String key = "test_degrade_exception_count"; ClusterNode cn = mock(ClusterNode.class); - when(cn.totalException()).thenReturn(10L); + when(cn.getDegradeMinusExceptionCount()).thenReturn(10L); ClusterBuilderSlot.getClusterNodeMap().put(new StringResourceWrapper(key, EntryType.IN), cn); Context context = mock(Context.class); @@ -135,13 +136,13 @@ public void testExceptionCountModeDegrade() throws Throwable { rule.setTimeWindow(2); rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION_COUNT); - when(cn.totalException()).thenReturn(4L); + when(cn.getDegradeMinusExceptionCount()).thenReturn(4L); // Will fail. assertFalse(rule.passCheck(context, node, 1)); // Restore from the degrade timeout. TimeUnit.SECONDS.sleep(3); - when(cn.totalException()).thenReturn(3L); + when(cn.getDegradeMinusExceptionCount()).thenReturn(3L); //When the fifth request ends, degrade will blow close. for (int i = 0; i < 10; i++) { assertTrue(rule.passCheck(context, node, 1)); From 0bc146fbf47b3c5ac591aa1e3c5aae5ff0539413 Mon Sep 17 00:00:00 2001 From: jiangli Date: Thu, 21 Mar 2019 17:39:01 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=E5=BE=AE=E8=B0=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java index cd32eec6a3..80ad3f68e9 100755 --- a/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java +++ b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java @@ -91,8 +91,6 @@ public void testExceptionRatioModeDegrade() throws Throwable { rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO); when(cn.successQps()).thenReturn(8d); - // when(cn.getDegradeSecondExceptionRatio()).thenReturn() - // Will fail. assertFalse(rule.passCheck(context, node, 1));