'How to load table and data from SQL file to MySQL DB using python
I have a sql file that looks as follows
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!50503 SET NAMES utf8mb4 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
--
-- Table structure for table `ML_3273`
--
DROP TABLE IF EXISTS `ML_3273`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `ML_3273` (
`id` bigint(20) NOT NULL,
`first_name` smallint(5) NOT NULL,
`second_name` date DEFAULT NULL,
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Dumping data for table `ML_3273`
--
LOCK TABLES `ML_3273` WRITE;
/*!40000 ALTER TABLE `ML_3273` DISABLE KEYS */;
/*!40000 ALTER TABLE `ML_3273` ENABLE KEYS */;
UNLOCK TABLES;
DROP TABLE IF EXISTS `acc_acc_tags`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `acc_accounting_rule_tags` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`rule_id` bigint(20) NOT NULL,
`code_id` int(11) NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `unique_tag_journal_entry` (`accounting_rule_id`,`code_id`),
KEY `code_id` (`code_id`),
CONSTRAINT `acc_acc_tags_ibfk_1` FOREIGN KEY (`rule_id`) REFERENCES acc_tags` (`id`),
CONSTRAINT `rule_tags_ibfk_3` FOREIGN KEY (`code_id`) REFERENCES `m_code` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Dumping data for table `acc_acc_tags`
--
LOCK TABLES `acc_acc_tags` WRITE;
/*!40000 ALTER TABLE `acc_acc_tags` DISABLE KEYS */;
/*!40000 ALTER TABLE `acc_acc_tags` ENABLE KEYS */;
UNLOCK TABLES;
I'm trying to load the data into MySQL db using Python in Airflow composer, I'm able to load most tables into the db but during the execution I get an error
LOAD ERROR: 'charmap' codec can't encode character '\u0130' in position 874924: character maps to undefined
I have added encoding='utf8' even switched with encoding='utf-8' but still getting the same issue.
Here is my code
def exec_sql_file(cursor, sql_file):
print ("\n[INFO] Executing SQL script file: '%s'" % (sql_file))
statement = ""
for line in open(sql_file, encoding='utf8'):
if re.match(r'--', line): # ignore sql comment lines
continue
if not re.search(r';$', line): # keep appending lines that don't end in ';'
statement = statement + line
else: # when you get a line ending in ';' then exec statement and reset for next statement
statement = statement + line
#print "\n\n[DEBUG] Executing SQL statement:\n%s" % (statement)
try:
cursor.execute(statement)
except (my.OperationalError, my.ProgrammingError) as e:
print ("\n[WARN] MySQLError during execute statement \n\tArgs: '%s'" % (str(e.args)))
statement = ""
@dag(schedule_interval=None, default_args=default_args)
def load_to_mysql_dag():
@task
def load_to_mysql(file_path=FILE_PATH):
try:
connection = mysql_hook.get_conn()
cursor = connection.cursor()
exec_sql_file(cursor=cursor, sql_file=file_path)
print('LOAD TO MYSQL COMPLETE')
except Exception as xerror:
print("LOAD ERROR: ", xerror)
Anything I'm missing?
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|
