'Insert data from one table to another table in chunks - MySQL
I have been trying to load data from one table that has 100 million + rows into a new table based on a filter. I have tried to load using the procedure below but for some reason it just sits there with no result or activity in processlist.
I have been trying to insert data based on updated date but end up either inserting the same data each time running into a PK violation or no update.
Please advise what's the best way to load this data into a new table. Thanks
USE `mydb`;
DROP procedure IF EXISTS `insert_data`;
DELIMITER $$
USE `ods`$$
CREATE PROCEDURE `insert_data` ()
BEGIN
DECLARE i INT DEFAULT 0;
DECLARE limitSize INT DEFAULT 2000;
DECLARE maxupdate datetime;
SET maxupdate = (SELECT MAX(updated) FROM test_data);
WHILE i <= maxId DO
START TRANSACTION;
INSERT INTO test_arc
SELECT * FROM test_data
WHERE check_time>='2022-02-05' and updated> i
ORDER BY updated ASC
LIMIT limitSize;
COMMIT;
SET i = i + limitSize;
END WHILE;
END$$
DELIMITER;
--My Tables:
--Actual Table
CREATE TABLE test_data (
UNIQUE_ID bigint(11) NOT NULL,
DEVICE_NAME varchar(128) NOT NULL DEFAULT '',
PARAMETER_NAME varchar(256) DEFAULT NULL,
GMT_OFFSET varchar(6) DEFAULT NULL,
CHECK_TIME datetime NOT NULL DEFAULT '1970-01-01 00:00:00',
OUTPUT_DATA text NOT NULL,
PERF_DATA text NOT NULL,
PRIMARY KEY (UNIQUE_ID)
);
--Archive Table
CREATE TABLE test_arc (
UNIQUE_ID bigint(11) NOT NULL,
DEVICE_NAME varchar(128) NOT NULL DEFAULT '',
PARAMETER_NAME varchar(256) DEFAULT NULL,
GMT_OFFSET varchar(6) DEFAULT NULL,
CHECK_TIME datetime NOT NULL DEFAULT '1970-01-01 00:00:00',
OUTPUT_DATA text NOT NULL,
PERF_DATA text NOT NULL,
PRIMARY KEY (UNIQUE_ID)
);
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|
