[feature] Issue#35: keep all FT files consistent when checkpoint lock is held

AliSQLBackup depends on tokudb_checkpoint_lock to implement hot backup, but
invoking ft_close() to a FT file during backup will break the consistency
between this FT and the others in the final backup directory, even when
checkpoint lock is held, becasue the header of FT is updated by ft_close().

This patch checks whether we are in backup before closing FT, if in backup,
don't close FT.

Author: AliSQL

mysql-test/suite/rds/r/tokudb.result

@@ -277,10 +277,10 @@ MAX(c1)
 3
 show status like '%tokudb_rows%';
 Variable_name	Value
-Tokudb_rows_inserted	41
-Tokudb_rows_read	98
-Tokudb_rows_deleted	8
-Tokudb_rows_updated	19
+Tokudb_rows_inserted	####
+Tokudb_rows_read	####
+Tokudb_rows_deleted	####
+Tokudb_rows_updated	####
 show variables like 'tokudb_cpu_nums';
 Variable_name	Value
 tokudb_cpu_nums	0

mysql-test/suite/rds/r/tokudb_backup_lock.result

@@ -0,0 +1,113 @@
+=====1) hotbackup test=====
+CREATE TABLE test.t1 (c1 INT PRIMARY KEY, c2 INT, c3 INT) ENGINE=TOKUDB;
+INSERT INTO test.t1 VALUES(1111,1111,11111);
+CREATE TABLE test.t11 (c1 INT PRIMARY KEY, c2 INT, c3 INT) ENGINE=TOKUDB;
+CREATE TABLE test.t2 (c1 INT PRIMARY KEY, c2 INT) ENGINE=TOKUDB;
+INSERT INTO test.t2 VALUES(1122,1122);
+SET GLOBAL tokudb_checkpoint_on_flush_logs=ON;
+FLUSH LOGS;
+SET GLOBAL tokudb_checkpoint_on_flush_logs=OFF;
+INSERT INTO test.t1 VALUES(1,1,1);
+SET GLOBAL TOKUDB_CHECKPOINT_LOCK=ON;
+CREATE TABLE test.t22 (c1 INT PRIMARY KEY, c2 INT) ENGINE=TOKUDB;
+INSERT INTO test.t22 VALUES(1,1);
+INSERT INTO test.t1 VALUES(2,2,2);
+FLUSH TABLES WITH READ LOCK;
+UNLOCK TABLES;
+conn2 write(22,22,22), this row can't visible in new backup instance
+INSERT INTO test.t1 VALUES(22222,22222,22222);
+FLUSH TABLES;
+ALTER TABLE test.t11 RENAME test.t111;
+DROP TABLE test.t22;
+DROP TABLE test.t111;
+SET GLOBAL TOKUDB_CHECKPOINT_LOCK=OFF;
+shutdown server
+SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TOKUDB_FILE_MAP WHERE TABLE_SCHEMA='test' AND TABLE_DICTIONARY_NAME='main';
+TABLE_NAME
+t1
+t11
+t2
+t22
+OPTIMIZE TABLE test.t1;
+Table	Op	Msg_type	Msg_text
+test.t1	optimize	note	Table does not support optimize, doing recreate + analyze instead
+test.t1	optimize	status	OK
+SELECT * FROM test.t1;
+c1	c2	c3
+1	1	1
+2	2	2
+1111	1111	11111
+SELECT * FROM test.t2;
+c1	c2
+1122	1122
+DROP TABLE test.t1;
+DROP TABLE test.t2;
+=====2) crash and recover test=====
+CREATE TABLE test.t2 (c1 INT, c2 INT, PRIMARY KEY(c1), CLUSTERING KEY(c2)) ENGINE=TOKUDB;
+1
+SELECT COUNT(*) FROM test.t2;
+COUNT(*)
+1000
+DROP TABLE test.t2;
+=====3) create empty table and shutdown with checkpoint lock=====
+CREATE TABLE test.t1 (c1 INT, c2 INT, PRIMARY KEY(c1), CLUSTERING KEY(c2)) ENGINE=TOKUDB;
+CREATE TABLE test.t2 (c1 INT) ENGINE=TOKUDB;
+shutdown server
+SELECT * FROM test.t1;
+c1	c2
+SELECT * FROM test.t2;
+c1
+CREATE TABLE test.t3 (c1 INT) ENGINE=TOKUDB;
+INSERT INTO test.t3 VALUES(1);
+shutdown server
+SELECT * FROM test.t3;
+c1
+1
+DROP TABLE test.t1;
+DROP TABLE test.t2;
+DROP TABLE test.t3;
+=====4) alter table with checkpoint lock=====
+CREATE TABLE test.t1 (c1 INT PRIMARY KEY, c2 INT) ENGINE=TOKUDB;
+INSERT INTO test.t1 VALUES(1,1);
+SET GLOBAL TOKUDB_CHECKPOINT_LOCK=ON;
+FLUSH TABLES WITH READ LOCK;
+UNLOCK TABLES;
+ALTER TABLE test.t1 ADD COLUMN `c3` INT;
+INSERT INTO test.t1 VALUES(2,2,3);
+SET GLOBAL TOKUDB_CHECKPOINT_LOCK=OFF;
+shutdown server
+OPTIMIZE TABLE test.t1;
+Table	Op	Msg_type	Msg_text
+test.t1	optimize	note	Table does not support optimize, doing recreate + analyze instead
+test.t1	optimize	status	OK
+SELECT * FROM test.t1;
+c1	c2
+1	1
+DROP TABLE test.t1;
+=====5) kill insert/alter/add index when checkpoint lock ON=====
+shutdown server
+CREATE TABLE test.t1 (c1 INT PRIMARY KEY, c2 INT) ENGINE=TOKUDB;
+SET GLOBAL TOKUDB_CHECKPOINT_LOCK=ON;
+CREATE TABLE test.t3 (c1 INT PRIMARY KEY, c2 INT) ENGINE=TOKUDB;
+ALTER TABLE test.t1 ADD COLUMN `c3` INT;
+INSERT INTO test.t1 VALUES(2,2,3);
+ALTER TABLE test.t3 ADD INDEX idx_c1c2(c1, c2);
+1
+SHOW CREATE TABLE test.t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` int(11) NOT NULL,
+  `c2` int(11) DEFAULT NULL,
+  `c3` int(11) DEFAULT NULL,
+  PRIMARY KEY (`c1`)
+) ENGINE=TokuDB DEFAULT CHARSET=latin1
+SHOW CREATE TABLE test.t3;
+Table	Create Table
+t3	CREATE TABLE `t3` (
+  `c1` int(11) NOT NULL,
+  `c2` int(11) DEFAULT NULL,
+  PRIMARY KEY (`c1`),
+  KEY `idx_c1c2` (`c1`,`c2`)
+) ENGINE=TokuDB DEFAULT CHARSET=latin1
+DROP TABLE test.t1;
+DROP TABLE test.t3;

mysql-test/suite/rds/t/tokudb.test

@@ -182,6 +182,7 @@ SELECT MAX(c1) FROM t3;
 --let $innodb_status =`show engine tokudb status`
 --sleep 2
 --let $innodb_status =`show engine tokudb status`
+--replace_column 2 ####
 show status like '%tokudb_rows%';
 show variables like 'tokudb_cpu_nums';