plan: maintain HistColl in Selection and Projection's statsInfo.

plan/cbo_test.go

@@ -690,6 +690,28 @@ func (s *testAnalyzeSuite) TestInconsistentEstimation(c *C) {
 		))
 }
 
+func (s *testAnalyzeSuite) TestHistogramInPlan(c *C) {
+	defer testleak.AfterTest(c)()
+	store, dom, err := newStoreWithBootstrap()
+	c.Assert(err, IsNil)
+	tk := testkit.NewTestKit(c, store)
+	defer func() {
+		dom.Close()
+		store.Close()
+	}()
+	tk.MustExec("use test")
+	tk.MustExec("set @@session.tidb_optimizer_selectivity_level=1")
+	tk.MustExec(`create table t(a enum("a", "b", "c"), index idx(a))`)
+	tk.MustExec(`insert into t values("a"), ("a"), ("a"), ("a"), ("b"), ("b"), ("b"), ("C")`)
+	tk.MustExec("analyze table t")
+	tk.MustQuery("explain select * from t where a = 'c'").
+		Check(testkit.Rows(
+			"Selection_5 1.00 root eq(test.t.a, \"c\")",
+			"└─TableReader_7 8.00 root data:TableScan_6",
+			"  └─TableScan_6 8.00 cop table:t, range:[-inf,+inf], keep order:false",
+		))
+}
+
 func newStoreWithBootstrap() (kv.Storage, *domain.Domain, error) {
 	store, err := mockstore.NewMockTikvStore()
 	if err != nil {

plan/logical_plans.go

@@ -406,7 +406,7 @@ func (ds *DataSource) deriveTablePathStats(path *accessPath) (bool, error) {
 // deriveIndexPathStats will fulfill the information that the accessPath need.
 // And it will check whether this index is full matched by point query. We will use this check to
 // determine whether we remove other paths or not.
-func (ds *DataSource) deriveIndexPathStats(path *accessPath) (bool, error) {
+func (ds *DataSource) deriveIndexPathStats(path *accessPath, histColl *statistics.HistColl) (bool, error) {
 	var err error
 	sc := ds.ctx.GetSessionVars().StmtCtx
 	path.ranges = ranger.FullRange()
@@ -417,7 +417,7 @@ func (ds *DataSource) deriveIndexPathStats(path *accessPath) (bool, error) {
 		if err != nil {
 			return false, errors.Trace(err)
 		}
-		path.countAfterAccess, err = ds.stats.histColl.GetRowCountByIndexRanges(sc, path.index.ID, path.ranges)
+		path.countAfterAccess, err = histColl.GetRowCountByIndexRanges(sc, path.index.ID, path.ranges)
 		if err != nil {
 			return false, errors.Trace(err)
 		}
@@ -435,8 +435,8 @@ func (ds *DataSource) deriveIndexPathStats(path *accessPath) (bool, error) {
 	}
 	path.indexFilters, path.tableFilters = splitIndexFilterConditions(path.tableFilters, path.index.Columns, ds.tableInfo)
 	if corColInAccessConds {
-		idxHist, ok := ds.stats.histColl.Indices[path.index.ID]
-		if ok && !ds.stats.histColl.Pseudo {
+		idxHist, ok := histColl.Indices[path.index.ID]
+		if ok && !histColl.Pseudo {
 			path.countAfterAccess = idxHist.AvgCountPerValue(ds.statisticTable.Count)
 		} else {
 			path.countAfterAccess = ds.statisticTable.PseudoAvgCountPerValue()
@@ -448,7 +448,7 @@ func (ds *DataSource) deriveIndexPathStats(path *accessPath) (bool, error) {
 		path.countAfterAccess = math.Min(ds.stats.count/selectionFactor, float64(ds.statisticTable.Count))
 	}
 	if path.indexFilters != nil {
-		selectivity, err := ds.stats.histColl.Selectivity(ds.ctx, path.indexFilters)
+		selectivity, err := histColl.Selectivity(ds.ctx, path.indexFilters)
 		if err != nil {
 			log.Warnf("An error happened: %v, we have to use the default selectivity", err.Error())
 			selectivity = selectionFactor

plan/stats.go

@@ -28,7 +28,7 @@ type statsInfo struct {
 	count       float64
 	cardinality []float64
 
-	histColl statistics.HistColl
+	histColl *statistics.HistColl
 	// usePseudoStats indicates whether the statsInfo is calculated using the
 	// pseudo statistics on a table.
 	usePseudoStats bool
@@ -47,7 +47,6 @@ func (s *statsInfo) scale(factor float64) *statsInfo {
 	profile := &statsInfo{
 		count:          s.count * factor,
 		cardinality:    make([]float64, len(s.cardinality)),
-		histColl:       s.histColl,
 		usePseudoStats: s.usePseudoStats,
 	}
 	for i := range profile.cardinality {
@@ -110,11 +109,10 @@ func (p *baseLogicalPlan) deriveStats() (*statsInfo, error) {
 	return profile, nil
 }
 
-func (ds *DataSource) getStatsByFilter(conds expression.CNFExprs) *statsInfo {
-	profile := &statsInfo{
+func (ds *DataSource) getStatsByFilter(conds expression.CNFExprs) (profile *statsInfo, histColl *statistics.HistColl) {
+	profile = &statsInfo{
 		count:          float64(ds.statisticTable.Count),
 		cardinality:    make([]float64, len(ds.Columns)),
-		histColl:       ds.statisticTable.GenerateHistCollFromColumnInfo(ds.Columns, ds.schema.Columns),
 		usePseudoStats: ds.statisticTable.Pseudo,
 	}
 	for i, col := range ds.Columns {
@@ -126,21 +124,22 @@ func (ds *DataSource) getStatsByFilter(conds expression.CNFExprs) *statsInfo {
 			profile.cardinality[i] = profile.count * distinctFactor
 		}
 	}
-	ds.stats = profile
-	selectivity, err := profile.histColl.Selectivity(ds.ctx, conds)
+	histColl = ds.statisticTable.GenerateHistCollFromColumnInfo(ds.Columns, ds.schema.Columns)
+	selectivity, err := histColl.Selectivity(ds.ctx, conds)
 	if err != nil {
 		log.Warnf("An error happened: %v, we have to use the default selectivity", err.Error())
 		selectivity = selectionFactor
 	}
-	return profile.scale(selectivity)
+	return profile.scale(selectivity), histColl
 }
 
 func (ds *DataSource) deriveStats() (*statsInfo, error) {
 	// PushDownNot here can convert query 'not (a != 1)' to 'a = 1'.
 	for i, expr := range ds.pushedDownConds {
 		ds.pushedDownConds[i] = expression.PushDownNot(nil, expr, false)
 	}
-	ds.stats = ds.getStatsByFilter(ds.pushedDownConds)
+	statsInfo, histColl := ds.getStatsByFilter(ds.pushedDownConds)
+	ds.stats = statsInfo
 	for _, path := range ds.possibleAccessPaths {
 		if path.isTablePath {
 			noIntervalRanges, err := ds.deriveTablePathStats(path)
@@ -155,7 +154,7 @@ func (ds *DataSource) deriveStats() (*statsInfo, error) {
 			}
 			continue
 		}
-		noIntervalRanges, err := ds.deriveIndexPathStats(path)
+		noIntervalRanges, err := ds.deriveIndexPathStats(path, histColl)
 		if err != nil {
 			return nil, errors.Trace(err)
 		}
@@ -166,6 +165,8 @@ func (ds *DataSource) deriveStats() (*statsInfo, error) {
 			break
 		}
 	}
+	histColl.Count = int64(ds.stats.count)
+	ds.stats.histColl = histColl
 	return ds.stats, nil
 }
 
@@ -174,7 +175,23 @@ func (p *LogicalSelection) deriveStats() (*statsInfo, error) {
 	if err != nil {
 		return nil, errors.Trace(err)
 	}
-	p.stats = childProfile.scale(selectionFactor)
+	factor := selectionFactor
+	if p.ctx.GetSessionVars().OptimizerSelectivityLevel == 1 && childProfile.histColl != nil {
+		factor, err = childProfile.histColl.Selectivity(p.ctx, p.Conditions)
+		if err != nil {
+			return nil, errors.Trace(err)
+		}
+	}
+	p.stats = childProfile.scale(factor)
+	if childProfile.histColl != nil {
+		p.stats.histColl = &statistics.HistColl{
+			Count:         int64(p.stats.count),
+			Columns:       childProfile.histColl.Columns,
+			Indices:       childProfile.histColl.Indices,
+			Idx2ColumnIDs: childProfile.histColl.Idx2ColumnIDs,
+			ColID2IdxID:   childProfile.histColl.ColID2IdxID,
+		}
+	}
 	return p.stats, nil
 }
 
@@ -254,9 +271,58 @@ func (p *LogicalProjection) deriveStats() (*statsInfo, error) {
 		cols := expression.ExtractColumns(expr)
 		p.stats.cardinality[i] = getCardinality(cols, p.children[0].Schema(), childProfile)
 	}
+	// If we enables the enhance selectivity we'll try to maintain the histogram.
+	if p.ctx.GetSessionVars().OptimizerSelectivityLevel >= 1 && !childProfile.histColl.Pseudo {
+		p.deriveHistStats(childProfile)
+	}
 	return p.stats, nil
 }
 
+func (p *LogicalProjection) deriveHistStats(childProfile *statsInfo) {
+	colHistMap := make(map[int64]*statistics.Column)
+	colIDMap := make(map[int64]int64)
+	childHist := childProfile.histColl
+	for i, expr := range p.Exprs {
+		col, ok := expr.(*expression.Column)
+		if !ok {
+			continue
+		}
+		colHist, ok := childHist.Columns[col.UniqueID]
+		if !ok {
+			continue
+		}
+		colHistMap[p.schema.Columns[i].UniqueID] = colHist
+		colIDMap[col.UniqueID] = p.schema.Columns[i].UniqueID
+	}
+
+	colID2IdxID := make(map[int64]int64)
+	idx2ColumnIDs := make(map[int64][]int64)
+	idxHistMap := make(map[int64]*statistics.Index)
+	for id, colIDs := range childHist.Idx2ColumnIDs {
+		newIDList := make([]int64, 0, len(colIDs))
+		for _, id := range colIDs {
+			if newID, ok := colIDMap[id]; ok {
+				newIDList = append(newIDList, newID)
+				continue
+			}
+			break
+		}
+		if len(newIDList) == 0 {
+			continue
+		}
+		colID2IdxID[newIDList[0]] = id
+		idx2ColumnIDs[id] = newIDList
+		idxHistMap[id] = childHist.Indices[id]
+	}
+	p.stats.histColl = &statistics.HistColl{
+		Columns:       colHistMap,
+		Indices:       idxHistMap,
+		Idx2ColumnIDs: idx2ColumnIDs,
+		ColID2IdxID:   colID2IdxID,
+		Count:         childHist.Count,
+	}
+}
+
 func (la *LogicalAggregation) deriveStats() (*statsInfo, error) {
 	childProfile, err := la.children[0].deriveStats()
 	if err != nil {

sessionctx/variable/varsutil.go

@@ -293,7 +293,7 @@ func ValidateSetSystemVar(vars *SessionVars, name string, value string) (string,
 		TiDBDistSQLScanConcurrency,
 		TiDBIndexSerialScanConcurrency, TiDBDDLReorgWorkerCount,
 		TiDBBackoffLockFast, TiDBMaxChunkSize,
-		TiDBDMLBatchSize, TiDBOptimizerSelectivityLevel:
+		TiDBDMLBatchSize:
 		v, err := strconv.Atoi(value)
 		if err != nil {
 			return value, ErrWrongTypeForVar.GenByArgs(name)

statistics/table.go

@@ -471,7 +471,7 @@ func getOrdinalOfRangeCond(sc *stmtctx.StatementContext, ran *ranger.Range) int
 }
 
 // GenerateHistCollFromColumnInfo generates a new HistColl whose ColID2IdxID and IdxID2ColIDs is built from the given parameter.
-func (coll *HistColl) GenerateHistCollFromColumnInfo(infos []*model.ColumnInfo, columns []*expression.Column) HistColl {
+func (coll *HistColl) GenerateHistCollFromColumnInfo(infos []*model.ColumnInfo, columns []*expression.Column) *HistColl {
 	newColHistMap := make(map[int64]*Column)
 	colInfoID2UniqueID := make(map[int64]int64)
 	colNames2UniqueID := make(map[string]int64)
@@ -511,7 +511,7 @@ func (coll *HistColl) GenerateHistCollFromColumnInfo(infos []*model.ColumnInfo,
 		newIdxHistMap[idxHist.ID] = idxHist
 		idx2Columns[idxHist.ID] = ids
 	}
-	newColl := HistColl{
+	newColl := &HistColl{
 		PhysicalID:     coll.PhysicalID,
 		HavePhysicalID: coll.HavePhysicalID,
 		Pseudo:         coll.Pseudo,