Merge pull request #2820 from bakdata/feature/masked-add-close-to-now

truncate positive infinity to today, don't output if negative infinity is part of a count
ingef · Nov 24, 2022 · 2b872f9 · 2b872f9
2 parents dd4b35b + 4d24f69
commit 2b872f9
Show file tree

Hide file tree

Showing 12 changed files with 267 additions and 211 deletions.
diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/CDateSet.java b/backend/src/main/java/com/bakdata/conquery/models/common/CDateSet.java
@@ -271,7 +271,11 @@ private void putRange(CDateRange range) {
 	}
 
 
-	public void maskedAdd(CDateRange toAdd, CDateSet mask){
+	public void maskedAdd(CDateRange toAdd, CDateSet mask) {
+		maskedAdd(toAdd, mask, CDateRange.POSITIVE_INFINITY);
+	}
+
+	public void maskedAdd(CDateRange toAdd, CDateSet mask, int truncateMax){
 		if(mask.isEmpty()){
 			return;
 		}
@@ -317,27 +321,29 @@ public void maskedAdd(CDateRange toAdd, CDateSet mask){
 
 			search = mask.rangesByLowerBound.higherKey(search);
 
-			int min = range.getMinValue();
-			int max = range.getMaxValue();
+			int lowerBound = range.getMinValue();
+			int upperBound = range.getMaxValue();
 
-			if(max < toAdd.getMinValue()){
+			if(upperBound < toAdd.getMinValue()){
 				continue;
 			}
 
-			if(min < toAdd.getMinValue()){
-				min = toAdd.getMinValue();
+			if(lowerBound < toAdd.getMinValue()){
+				lowerBound = toAdd.getMinValue();
 			}
 
-			if(max > toAdd.getMaxValue()){
-				max = toAdd.getMaxValue();
+			if(upperBound > toAdd.getMaxValue()){
+				upperBound = toAdd.getMaxValue();
 			}
 
+			upperBound = Math.min(upperBound, truncateMax);
+
 			// value was not contained
-			if(min > max){
+			if(lowerBound > upperBound){
 				continue;
 			}
 
-			add(CDateRange.of(min, max));
+			add(CDateRange.of(lowerBound, upperBound));
 		}
 	}
 

diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutionContext.java b/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutionContext.java
@@ -1,14 +1,16 @@
 package com.bakdata.conquery.models.query;
 
+import java.time.LocalDate;
 import java.util.List;
 import java.util.Optional;
 
 import com.bakdata.conquery.io.storage.ModificationShieldedWorkerStorage;
+import com.bakdata.conquery.models.common.CDate;
 import com.bakdata.conquery.models.common.CDateSet;
-import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.datasets.SecondaryIdDescription;
 import com.bakdata.conquery.models.datasets.Table;
+import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.events.Bucket;
 import com.bakdata.conquery.models.events.BucketManager;
 import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId;
@@ -44,6 +46,8 @@ public class QueryExecutionContext {
 	 */
 	private SecondaryIdDescription activeSecondaryId = null;
 
+	private final int today = CDate.ofLocalDate(LocalDate.now());
+
 	public List<Bucket> getEntityBucketsForTable(Entity entity, Table table) {
 		return bucketManager.getEntityBucketsForTable(entity, table);
 	}

diff --git a/...com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateUnionAggregator.java b/...com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateUnionAggregator.java
@@ -20,13 +20,16 @@ public class DateUnionAggregator extends SingleColumnAggregator<CDateSet> {
 	private CDateSet set = CDateSet.create();
 	private CDateSet dateRestriction;
 
+	private int realUpperBound;
+
 	public DateUnionAggregator(Column column) {
 		super(column);
 	}
 
 	@Override
 	public void init(Entity entity, QueryExecutionContext context) {
 		set.clear();
+		realUpperBound = context.getToday();
 	}
 
 	@Override
@@ -40,13 +43,9 @@ public void acceptEvent(Bucket bucket, int event) {
 			return;
 		}
 
-		CDateRange value = bucket.getAsDateRange(event, getColumn());
-		//otherwise the result would be something weird
-		if (value.isOpen()) {
-			return;
-		}
+		final CDateRange value = bucket.getAsDateRange(event, getColumn());
 
-		set.maskedAdd(value, dateRestriction);
+		set.maskedAdd(value, dateRestriction, realUpperBound);
 	}
 
 	@Override

diff --git a/...m/bakdata/conquery/models/query/queryplan/aggregators/specific/DurationSumAggregator.java b/...m/bakdata/conquery/models/query/queryplan/aggregators/specific/DurationSumAggregator.java
@@ -1,5 +1,6 @@
 package com.bakdata.conquery.models.query.queryplan.aggregators.specific;
 
+import com.bakdata.conquery.models.common.CDate;
 import com.bakdata.conquery.models.common.CDateSet;
 import com.bakdata.conquery.models.common.daterange.CDateRange;
 import com.bakdata.conquery.models.datasets.Column;
@@ -20,13 +21,16 @@ public class DurationSumAggregator extends SingleColumnAggregator<Long> {
 	private CDateSet set = CDateSet.create();
 	private CDateSet dateRestriction;
 
+	private int realUpperBound;
+
 	public DurationSumAggregator(Column column) {
 		super(column);
 	}
 
 	@Override
 	public void init(Entity entity, QueryExecutionContext context) {
 		set.clear();
+		realUpperBound = context.getToday();
 	}
 
 	@Override
@@ -42,17 +46,15 @@ public void acceptEvent(Bucket bucket, int event) {
 
 		final CDateRange value = bucket.getAsDateRange(event, getColumn());
 
-		if (value.isOpen()) {
-			return;
-		}
-
-
-		set.maskedAdd(value, dateRestriction);
+		set.maskedAdd(value, dateRestriction, realUpperBound);
 	}
 
 	@Override
 	public Long createAggregationResult() {
-		return set.isEmpty() ? null : set.countDays();
+		if (set.isEmpty() || CDate.isNegativeInfinity(set.getMinValue())) {
+			return null;
+		}
+		return set.countDays();
 	}
 
 	@Override

diff --git a/...data/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java b/...data/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java
@@ -21,17 +21,18 @@
 @ToString(onlyExplicitlyIncluded = true)
 public class EventDurationSumAggregator extends Aggregator<Long> {
 
-	private Optional<Aggregator<CDateSet>> queryDateAggregator = Optional.empty();
 	private final CDateSet set = CDateSet.create();
-
+	private Optional<Aggregator<CDateSet>> queryDateAggregator = Optional.empty();
 	@CheckForNull
 	private CDateSet dateRestriction;
 	@CheckForNull
 	private Column validityDateColumn;
+	private int realUpperBound;
 
 	@Override
 	public void init(Entity entity, QueryExecutionContext context) {
 		set.clear();
+		realUpperBound = context.getToday();
 	}
 
 	@Override
@@ -53,12 +54,8 @@ public void acceptEvent(Bucket bucket, int event) {
 
 		final CDateRange value = bucket.getAsDateRange(event, validityDateColumn);
 
-		if (value.isOpen()) {
-			return;
-		}
-
 
-		set.maskedAdd(value, dateRestriction);
+		set.maskedAdd(value, dateRestriction, realUpperBound);
 	}
 
 	@Override

diff --git a/...a/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java b/...a/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java
@@ -33,6 +33,8 @@ public class QuarterAggregator extends Aggregator<String> {
 
 	private Column column;
 
+	private int realUpperBound;
+
 	public QuarterAggregator(TemporalSamplerFactory samplerFactory) {
 		this.samplerFactory = samplerFactory;
 	}
@@ -41,6 +43,7 @@ public QuarterAggregator(TemporalSamplerFactory samplerFactory) {
 	public void init(Entity entity, QueryExecutionContext context) {
 		set.clear();
 		sampler = samplerFactory.sampler();
+		realUpperBound = context.getToday();
 	}
 
 	@Override
@@ -57,11 +60,7 @@ public void acceptEvent(Bucket bucket, int event) {
 
 		final CDateRange value = bucket.getAsDateRange(event, getColumn());
 
-		if (value.isOpen()) {
-			return;
-		}
-
-		set.maskedAdd(value, dateRestriction);
+		set.maskedAdd(value, dateRestriction, realUpperBound);
 	}
 
 	@Override
@@ -70,13 +69,19 @@ public String createAggregationResult() {
 			return null;
 		}
 
-		final OptionalInt sampled = sampler.sample(set);
+		final OptionalInt maybeSampled = sampler.sample(set);
+
+		if (maybeSampled.isEmpty()) {
+			return null;
+		}
+
+		final int sampled = maybeSampled.getAsInt();
 
-		if (sampled.isEmpty()) {
+		if (CDate.isNegativeInfinity(sampled) || CDate.isPositiveInfinity(sampled)) {
 			return null;
 		}
 
-		final LocalDate date = CDate.toLocalDate(sampled.getAsInt());
+		final LocalDate date = CDate.toLocalDate(sampled);
 		final int quarter = QuarterUtils.getQuarter(date);
 		final int year = date.getYear();
 

diff --git a/...va/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java b/...va/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java
@@ -37,21 +37,20 @@ public void nextTable(QueryExecutionContext ctx, Table table) {
 
 	@Override
 	public void acceptEvent(Bucket bucket, int event) {
-		if (currentColumn != null && bucket.has(event, currentColumn)) {
-			set.maskedAdd(bucket.getAsDateRange(event, currentColumn), dateRestriction);
+		if (currentColumn == null || !bucket.has(event, currentColumn)) {
+			set.addAll(dateRestriction);
 			return;
 		}
 
-		if(!dateRestriction.isEmpty()) {
-			set.addAll(dateRestriction);
-		}
+		set.maskedAdd(bucket.getAsDateRange(event, currentColumn), dateRestriction);
 	}
 
 	/**
 	 * Helper method to insert dates from outside.
+	 *
 	 * @param other CDateSet to be included.
 	 */
-	public void merge(CDateSet other){
+	public void merge(CDateSet other) {
 		set.addAll(other);
 	}