Merge branch 'master' into dev-digest-log

CONTRIBUTING.md

@@ -6,12 +6,12 @@ may be different than many other projects you have been involved in. This docume
 ## What is a Contributor?
 
 A Contributor refers to the person who contributes to the following projects:
-- TiDB: https://github.com/pingcap/tidb 
-- TiKV: https://github.com/tikv/tikv 
-- TiSpark: https://github.com/pingcap/tispark 
-- PD: https://github.com/pingcap/pd 
-- Docs: https://github.com/pingcap/docs 
-- Docs-cn: https://github.com/pingcap/docs-cn 
+- TiDB: https://github.com/pingcap/tidb
+- TiKV: https://github.com/tikv/tikv
+- TiSpark: https://github.com/pingcap/tispark
+- PD: https://github.com/pingcap/pd
+- Docs: https://github.com/pingcap/docs
+- Docs-cn: https://github.com/pingcap/docs-cn
 
 ## How to become a TiDB Contributor?
 
@@ -61,7 +61,7 @@ TiDB is written in [Go](http://golang.org).
 If you don't have a Go development environment,
 please [set one up](http://golang.org/doc/code.html).
 
-The version of GO should be **1.11** or above.
+The version of GO should be **1.12** or above.
 
 After installation, there are two ways to build TiDB binary.
 
@@ -215,6 +215,8 @@ mysql -h127.0.0.1 -P4000 -uroot test --default-character-set utf8
 
 #### Run Test
 
+Run all tests
+
 ```sh
 # Run unit test to make sure all test passed.
 make dev
@@ -223,6 +225,14 @@ make dev
 make checklist
 ```
 
+You can also run a single test in a file. For example, if you want to run
+test `TestToInt64` in file `types/datum.go`, you can do something like
+
+```sh
+cd types
+GO111MODULE=on go test -check.f TestToInt64
+```
+
 ### Step 5: Keep your branch in sync
 
 ```sh
@@ -253,7 +263,7 @@ git push -f origin myfeature
 
 ### Step 8: Create a pull request
 
-1. Visit your fork at https://github.com/$user/tidb (replace `$user` obviously).
+1. Visit your fork at `https://github.com/$user/tidb` (replace `$user` obviously).
 2. Click the `Compare & pull request` button next to your `myfeature` branch.
 
 ### Step 9: Get a code review
@@ -273,33 +283,6 @@ review.
 
 The coding style suggested by the Golang community is used in TiDB. See the [style doc](https://github.com/golang/go/wiki/CodeReviewComments) for details.
 
-## Commit message style
-
-Please follow this style to make TiDB easy to review, maintain and develop.
-
-```
-<subsystem>: <what changed>
-<BLANK LINE>
-<why this change was made>
-<BLANK LINE>
-<footer>(optional)
-```
-
-The first line is the subject and should be no longer than 70 characters, the
-second line is always blank, and other lines should be wrapped at 80 characters.
-This allows the message to be easier to read on GitHub as well as in various
-git tools.
-
-If the change affects more than one subsystem, you can use comma to separate them like `util/codec,util/types:`.
-
-If the change affects many subsystems, you can use ```*``` instead, like ```*:```.
-
-For the why part, if no specific reason for the change,
-you can use one of some generic reasons like "Improve documentation.",
-"Improve performance.", "Improve robustness.", "Improve test coverage."
+## Commit message and Pull Request style
 
-[Os X GNU tools]: https://www.topbug.net/blog/2013/04/14/install-and-use-gnu-command-line-tools-in-mac-os-x
-[go-1.8]: https://blog.golang.org/go1.8
-[go-workspace]: https://golang.org/doc/code.html#Workspaces
-[issue]: https://github.com/pingcap/tidb/issues
-[mercurial]: http://mercurial.selenic.com/wiki/Download
+See the [Commit Message and Pull Request Style](https://github.com/pingcap/community/blob/master/commit-message-pr-style.md)

cmd/explaintest/r/topn_push_down.result

@@ -186,9 +186,8 @@ Projection_13	0.00	root	te.expect_time
         └─IndexScan_101	10.00	cop	table:p, index:relate_id, range: decided by [tr.id], keep order:false, stats:pseudo
 desc select 1 as a from dual order by a limit 1;
 id	count	task	operator info
-Projection_7	1.00	root	1
-└─Limit_8	1.00	root	offset:0, count:1
-  └─TableDual_11	1.00	root	rows:1
+Projection_6	1.00	root	1
+└─TableDual_7	1.00	root	rows:1
 drop table if exists t1;
 drop table if exists t2;
 create table t1(a bigint, b bigint);

cmd/explaintest/r/topn_pushdown.result

@@ -0,0 +1,9 @@
+explain select * from ((select 4 as a) union all (select 33 as a)) tmp order by a desc limit 1;
+id	count	task	operator info
+TopN_17	1.00	root	tmp.a:desc, offset:0, count:1
+└─Union_21	2.00	root	
+  ├─Projection_22	1.00	root	cast(a)
+  │ └─Projection_23	1.00	root	4
+  │   └─TableDual_24	1.00	root	rows:1
+  └─Projection_26	1.00	root	33
+    └─TableDual_27	1.00	root	rows:1

cmd/explaintest/t/topn_pushdown.test

@@ -0,0 +1 @@
+explain select * from ((select 4 as a) union all (select 33 as a)) tmp order by a desc limit 1;

config/config.go

@@ -250,7 +250,7 @@ type TiKVClient struct {
 
 // Binlog is the config for binlog.
 type Binlog struct {
-	Enable       bool   `toml:"enable" json:"enable"`
+	Enable       string `toml:"enable" json:"enable"`
 	WriteTimeout string `toml:"write-timeout" json:"write-timeout"`
 	// If IgnoreError is true, when writing binlog meets error, TiDB would
 	// ignore the error.
@@ -343,6 +343,7 @@ var defaultConf = Config{
 		BatchWaitSize:     8,
 	},
 	Binlog: Binlog{
+		Enable:       "auto",
 		WriteTimeout: "15s",
 	},
 }

config/config.toml.example

@@ -258,8 +258,9 @@ enabled = true
 capacity = 2048000
 
 [binlog]
-# enable to write binlog.
-enable = false
+# Enable to write binlog. Values can be "on", "off" or "auto".
+# If value is "auto", will enable binlog according to the system variables 'tidb_log_bin'.
+enable = "auto"
 
 # WriteTimeout specifies how long it will wait for writing binlog to pump.
 write-timeout = "15s"

config/config_test.go

@@ -33,7 +33,7 @@ func TestT(t *testing.T) {
 
 func (s *testConfigSuite) TestConfig(c *C) {
 	conf := new(Config)
-	conf.Binlog.Enable = true
+	conf.Binlog.Enable = "auto"
 	conf.Binlog.IgnoreError = true
 	conf.TiKVClient.CommitTimeout = "10s"
 	conf.CheckMb4ValueInUtf8 = true
@@ -54,7 +54,7 @@ max-batch-size=128
 	c.Assert(conf.Load(configFile), IsNil)
 
 	// Test that the original value will not be clear by load the config file that does not contain the option.
-	c.Assert(conf.Binlog.Enable, Equals, true)
+	c.Assert(conf.Binlog.Enable, Equals, "auto")
 
 	c.Assert(conf.TiKVClient.CommitTimeout, Equals, "41s")
 	c.Assert(conf.TiKVClient.MaxBatchSize, Equals, uint(128))

ddl/db_test.go

@@ -1659,6 +1659,71 @@ func (s *testDBSuite) TestCreateTableWithLike(c *C) {
 	s.tk.MustExec("drop database ctwl_db1")
 }
 
+// TestCreateTableWithLike2 tests create table with like when refer table have non-public column/index.
+func (s *testDBSuite) TestCreateTableWithLike2(c *C) {
+	s.tk = testkit.NewTestKit(c, s.store)
+	s.tk.MustExec("use test_db")
+	s.tk.MustExec("drop table if exists t1,t2;")
+	defer s.tk.MustExec("drop table if exists t1,t2;")
+	s.tk.MustExec("create table t1 (a int, b int, c int, index idx1(c));")
+
+	tbl1 := testGetTableByName(c, s.s, "test_db", "t1")
+	doneCh := make(chan error, 2)
+	hook := &ddl.TestDDLCallback{}
+	hook.OnJobRunBeforeExported = func(job *model.Job) {
+		if job.Type != model.ActionAddColumn && job.Type != model.ActionDropColumn && job.Type != model.ActionAddIndex && job.Type != model.ActionDropIndex {
+			return
+		}
+		if job.TableID != tbl1.Meta().ID {
+			return
+		}
+		if job.SchemaState == model.StateDeleteOnly {
+			go backgroundExec(s.store, "create table t2 like t1", doneCh)
+		}
+	}
+	originalHook := s.dom.DDL().GetHook()
+	defer s.dom.DDL().(ddl.DDLForTest).SetHook(originalHook)
+	s.dom.DDL().(ddl.DDLForTest).SetHook(hook)
+
+	// create table when refer table add column
+	s.tk.MustExec("alter table t1 add column d int")
+	checkTbl2 := func() {
+		err := <-doneCh
+		c.Assert(err, IsNil)
+		s.tk.MustExec("alter table t2 add column e int")
+		t2Info := testGetTableByName(c, s.s, "test_db", "t2")
+		c.Assert(len(t2Info.Meta().Columns), Equals, 4)
+		c.Assert(len(t2Info.Meta().Columns), Equals, len(t2Info.Cols()))
+	}
+	checkTbl2()
+
+	// create table when refer table drop column
+	s.tk.MustExec("drop table t2;")
+	s.tk.MustExec("alter table t1 drop column b;")
+	checkTbl2()
+
+	// create table when refer table add index
+	s.tk.MustExec("drop table t2;")
+	s.tk.MustExec("alter table t1 add index idx2(a);")
+	checkTbl2 = func() {
+		err := <-doneCh
+		c.Assert(err, IsNil)
+		s.tk.MustExec("alter table t2 add column e int")
+		tbl2 := testGetTableByName(c, s.s, "test_db", "t2")
+		c.Assert(len(tbl2.Meta().Columns), Equals, 4)
+		c.Assert(len(tbl2.Meta().Columns), Equals, len(tbl2.Cols()))
+		c.Assert(len(tbl2.Meta().Indices), Equals, 1)
+		c.Assert(tbl2.Meta().Indices[0].Name.L, Equals, "idx1")
+	}
+	checkTbl2()
+
+	// create table when refer table drop index.
+	s.tk.MustExec("drop table t2;")
+	s.tk.MustExec("alter table t1 drop index idx2;")
+	checkTbl2()
+
+}
+
 func (s *testDBSuite) TestCreateTable(c *C) {
 	s.tk.MustExec("use test")
 	s.tk.MustExec("CREATE TABLE `t` (`a` double DEFAULT 1.0 DEFAULT now() DEFAULT 2.0 );")

ddl/ddl_api.go

@@ -1008,10 +1008,7 @@ func (d *ddl) CreateTableWithLike(ctx sessionctx.Context, ident, referIdent ast.
 		return infoschema.ErrTableExists.GenWithStackByArgs(ident)
 	}
 
-	tblInfo := *referTbl.Meta()
-	tblInfo.Name = ident.Name
-	tblInfo.AutoIncID = 0
-	tblInfo.ForeignKeys = nil
+	tblInfo := buildTableInfoWithLike(ident, referTbl.Meta())
 	tblInfo.ID, err = d.genGlobalID()
 	if err != nil {
 		return errors.Trace(err)
@@ -1029,6 +1026,24 @@ func (d *ddl) CreateTableWithLike(ctx sessionctx.Context, ident, referIdent ast.
 	return errors.Trace(err)
 }
 
+func buildTableInfoWithLike(ident ast.Ident, referTblInfo *model.TableInfo) model.TableInfo {
+	tblInfo := *referTblInfo
+	// Check non-public column and adjust column offset.
+	newColumns := referTblInfo.Cols()
+	newIndices := make([]*model.IndexInfo, 0, len(tblInfo.Indices))
+	for _, idx := range tblInfo.Indices {
+		if idx.State == model.StatePublic {
+			newIndices = append(newIndices, idx)
+		}
+	}
+	tblInfo.Columns = newColumns
+	tblInfo.Indices = newIndices
+	tblInfo.Name = ident.Name
+	tblInfo.AutoIncID = 0
+	tblInfo.ForeignKeys = nil
+	return tblInfo
+}
+
 // BuildTableInfoFromAST builds model.TableInfo from a SQL statement.
 // The SQL string should be a create table statement.
 // Don't use this function to build a partitioned table.

ddl/ddl_worker_test.go

@@ -338,21 +338,15 @@ func checkCancelState(txn kv.Transaction, job *model.Job, test *testCancelJob) e
 	// If the action is adding index and the state is writing reorganization, it wants to test the case of cancelling the job when backfilling indexes.
 	// When the job satisfies this case of addIndexFirstReorg, the worker hasn't started to backfill indexes.
 	if test.cancelState == job.SchemaState && !addIndexFirstReorg {
-		if job.SchemaState == model.StateNone && job.State != model.JobStateDone && job.Type != model.ActionCreateTable && job.Type != model.ActionCreateSchema &&
-			job.Type != model.ActionRebaseAutoID && job.Type != model.ActionShardRowID && job.Type != model.ActionModifyColumn && job.Type != model.ActionAddForeignKey && job.Type != model.ActionDropForeignKey {
-			// If the schema state is none and is not equal to model.JobStateDone, we only test the job is finished.
-			// Unless the job is model.ActionCreateTable, model.ActionCreateSchema, model.ActionRebaseAutoID, we do the cancel anyway.
-		} else {
-			errs, err := admin.CancelJobs(txn, test.jobIDs)
-			if err != nil {
-				checkErr = errors.Trace(err)
-				return checkErr
-			}
-			// It only tests cancel one DDL job.
-			if !terror.ErrorEqual(errs[0], test.cancelRetErrs[0]) {
-				checkErr = errors.Trace(errs[0])
-				return checkErr
-			}
+		errs, err := admin.CancelJobs(txn, test.jobIDs)
+		if err != nil {
+			checkErr = errors.Trace(err)
+			return checkErr
+		}
+		// It only tests cancel one DDL job.
+		if !terror.ErrorEqual(errs[0], test.cancelRetErrs[0]) {
+			checkErr = errors.Trace(errs[0])
+			return checkErr
 		}
 	}
 	return checkErr
@@ -394,7 +388,16 @@ func buildCancelJobTests(firstID int64) []testCancelJob {
 		{act: model.ActionAddForeignKey, jobIDs: []int64{firstID + 19}, cancelRetErrs: noErrs, cancelState: model.StateNone},
 		{act: model.ActionAddForeignKey, jobIDs: []int64{firstID + 20}, cancelRetErrs: []error{admin.ErrCancelFinishedDDLJob.GenWithStackByArgs(firstID + 20)}, cancelState: model.StatePublic},
 		{act: model.ActionDropForeignKey, jobIDs: []int64{firstID + 21}, cancelRetErrs: noErrs, cancelState: model.StateNone},
-		{act: model.ActionDropForeignKey, jobIDs: []int64{firstID + 20}, cancelRetErrs: []error{admin.ErrCancelFinishedDDLJob.GenWithStackByArgs(firstID + 20)}, cancelState: model.StatePublic},
+		{act: model.ActionDropForeignKey, jobIDs: []int64{firstID + 22}, cancelRetErrs: []error{admin.ErrCancelFinishedDDLJob.GenWithStackByArgs(firstID + 22)}, cancelState: model.StatePublic},
+
+		{act: model.ActionRenameTable, jobIDs: []int64{firstID + 23}, cancelRetErrs: noErrs, cancelState: model.StateNone},
+		{act: model.ActionRenameTable, jobIDs: []int64{firstID + 24}, cancelRetErrs: []error{admin.ErrCancelFinishedDDLJob.GenWithStackByArgs(firstID + 24)}, cancelState: model.StatePublic},
+
+		{act: model.ActionModifyTableCharsetAndCollate, jobIDs: []int64{firstID + 25}, cancelRetErrs: noErrs, cancelState: model.StateNone},
+		{act: model.ActionModifyTableCharsetAndCollate, jobIDs: []int64{firstID + 26}, cancelRetErrs: []error{admin.ErrCancelFinishedDDLJob.GenWithStackByArgs(firstID + 26)}, cancelState: model.StatePublic},
+
+		{act: model.ActionTruncateTablePartition, jobIDs: []int64{firstID + 27}, cancelRetErrs: noErrs, cancelState: model.StateNone},
+		{act: model.ActionTruncateTablePartition, jobIDs: []int64{firstID + 28}, cancelRetErrs: []error{admin.ErrCancelFinishedDDLJob.GenWithStackByArgs(firstID + 28)}, cancelState: model.StatePublic},
 	}
 
 	return tests
@@ -443,12 +446,14 @@ func (s *testDDLSuite) TestCancelJob(c *C) {
 	defer d.Stop()
 	dbInfo := testSchemaInfo(c, d, "test_cancel_job")
 	testCreateSchema(c, testNewContext(d), d, dbInfo)
-
+	// create a partition table.
+	partitionTblInfo := testTableInfoWithPartition(c, d, "t_partition", 5)
 	// create table t (c1 int, c2 int, c3 int, c4 int, c5 int);
 	tblInfo := testTableInfo(c, d, "t", 5)
 	ctx := testNewContext(d)
 	err := ctx.NewTxn(context.Background())
 	c.Assert(err, IsNil)
+	testCreateTable(c, ctx, d, dbInfo, partitionTblInfo)
 	tableAutoID := int64(100)
 	shardRowIDBits := uint64(5)
 	tblInfo.AutoIncID = tableAutoID
@@ -674,6 +679,53 @@ func (s *testDDLSuite) TestCancelJob(c *C) {
 	c.Check(checkErr, IsNil)
 	changedTable = testGetTable(c, d, dbInfo.ID, tblInfo.ID)
 	c.Assert(len(changedTable.Meta().ForeignKeys), Equals, 0)
+
+	// test rename table failed caused by canceled.
+	test = &tests[20]
+	renameTableArgs := []interface{}{dbInfo.ID, model.NewCIStr("t2")}
+	doDDLJobErrWithSchemaState(ctx, d, c, dbInfo.ID, tblInfo.ID, test.act, renameTableArgs, &test.cancelState)
+	c.Check(checkErr, IsNil)
+	changedTable = testGetTable(c, d, dbInfo.ID, tblInfo.ID)
+	c.Assert(changedTable.Meta().Name.L, Equals, "t")
+
+	// test rename table successful.
+	test = &tests[21]
+	doDDLJobSuccess(ctx, d, c, dbInfo.ID, tblInfo.ID, test.act, renameTableArgs)
+	c.Check(checkErr, IsNil)
+	changedTable = testGetTable(c, d, dbInfo.ID, tblInfo.ID)
+	c.Assert(changedTable.Meta().Name.L, Equals, "t2")
+
+	// test modify table charset failed caused by canceled.
+	test = &tests[22]
+	modifyTableCharsetArgs := []interface{}{"utf8mb4", "utf8mb4_bin"}
+	doDDLJobErrWithSchemaState(ctx, d, c, dbInfo.ID, tblInfo.ID, test.act, modifyTableCharsetArgs, &test.cancelState)
+	c.Check(checkErr, IsNil)
+	changedTable = testGetTable(c, d, dbInfo.ID, tblInfo.ID)
+	c.Assert(changedTable.Meta().Charset, Equals, "utf8")
+	c.Assert(changedTable.Meta().Collate, Equals, "utf8_bin")
+
+	// test modify table charset successfully.
+	test = &tests[23]
+	doDDLJobSuccess(ctx, d, c, dbInfo.ID, tblInfo.ID, test.act, modifyTableCharsetArgs)
+	c.Check(checkErr, IsNil)
+	changedTable = testGetTable(c, d, dbInfo.ID, tblInfo.ID)
+	c.Assert(changedTable.Meta().Charset, Equals, "utf8mb4")
+	c.Assert(changedTable.Meta().Collate, Equals, "utf8mb4_bin")
+
+	// test truncate table partition failed caused by canceled.
+	test = &tests[24]
+	truncateTblPartitionArgs := []interface{}{partitionTblInfo.Partition.Definitions[0].ID}
+	doDDLJobErrWithSchemaState(ctx, d, c, dbInfo.ID, partitionTblInfo.ID, test.act, truncateTblPartitionArgs, &test.cancelState)
+	c.Check(checkErr, IsNil)
+	changedTable = testGetTable(c, d, dbInfo.ID, partitionTblInfo.ID)
+	c.Assert(changedTable.Meta().Partition.Definitions[0].ID == partitionTblInfo.Partition.Definitions[0].ID, IsTrue)
+
+	// test truncate table partition charset successfully.
+	test = &tests[25]
+	doDDLJobSuccess(ctx, d, c, dbInfo.ID, partitionTblInfo.ID, test.act, truncateTblPartitionArgs)
+	c.Check(checkErr, IsNil)
+	changedTable = testGetTable(c, d, dbInfo.ID, partitionTblInfo.ID)
+	c.Assert(changedTable.Meta().Partition.Definitions[0].ID == partitionTblInfo.Partition.Definitions[0].ID, IsFalse)
 }
 
 func (s *testDDLSuite) TestIgnorableSpec(c *C) {