'How to load table and data from SQL file to MySQL DB using python

I have a sql file that looks as follows

/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!50503 SET NAMES utf8mb4 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;

--
-- Table structure for table `ML_3273`
--

DROP TABLE IF EXISTS `ML_3273`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `ML_3273` (
  `id` bigint(20) NOT NULL,
  `first_name` smallint(5) NOT NULL,
  `second_name` date DEFAULT NULL,
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Dumping data for table `ML_3273`
--

LOCK TABLES `ML_3273` WRITE;
/*!40000 ALTER TABLE `ML_3273` DISABLE KEYS */;
/*!40000 ALTER TABLE `ML_3273` ENABLE KEYS */;
UNLOCK TABLES;

DROP TABLE IF EXISTS `acc_acc_tags`;
/*!40101 SET @saved_cs_client     = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `acc_accounting_rule_tags` (
  `id` bigint(20) NOT NULL AUTO_INCREMENT,
  `rule_id` bigint(20) NOT NULL,
  `code_id` int(11) NOT NULL,
  PRIMARY KEY (`id`),
  UNIQUE KEY `unique_tag_journal_entry` (`accounting_rule_id`,`code_id`),
  KEY `code_id` (`code_id`),
  CONSTRAINT `acc_acc_tags_ibfk_1` FOREIGN KEY (`rule_id`) REFERENCES acc_tags` (`id`),
  CONSTRAINT `rule_tags_ibfk_3` FOREIGN KEY (`code_id`) REFERENCES `m_code` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Dumping data for table `acc_acc_tags`
--

LOCK TABLES `acc_acc_tags` WRITE;
/*!40000 ALTER TABLE `acc_acc_tags` DISABLE KEYS */;
/*!40000 ALTER TABLE `acc_acc_tags` ENABLE KEYS */;
UNLOCK TABLES;

I'm trying to load the data into MySQL db using Python in Airflow composer, I'm able to load most tables into the db but during the execution I get an error

LOAD ERROR: 'charmap' codec can't encode character '\u0130' in position 874924: character maps to undefined

I have added encoding='utf8' even switched with encoding='utf-8' but still getting the same issue.

Here is my code

def exec_sql_file(cursor, sql_file):
    print ("\n[INFO] Executing SQL script file: '%s'" % (sql_file))
    statement = ""

    for line in open(sql_file, encoding='utf8'):
        if re.match(r'--', line):  # ignore sql comment lines
            continue
        if not re.search(r';$', line):  # keep appending lines that don't end in ';'
            statement = statement + line
        else:  # when you get a line ending in ';' then exec statement and reset for next statement
            statement = statement + line
            #print "\n\n[DEBUG] Executing SQL statement:\n%s" % (statement)
            try:
                cursor.execute(statement)
            except (my.OperationalError, my.ProgrammingError) as e:
                print ("\n[WARN] MySQLError during execute statement \n\tArgs: '%s'" % (str(e.args)))

            statement = ""

@dag(schedule_interval=None, default_args=default_args)
def load_to_mysql_dag():

    @task
    def load_to_mysql(file_path=FILE_PATH):
        try:
            connection = mysql_hook.get_conn()
            cursor = connection.cursor()
            exec_sql_file(cursor=cursor, sql_file=file_path)
            print('LOAD TO MYSQL COMPLETE')
        except Exception as xerror:
            print("LOAD ERROR: ", xerror)

Anything I'm missing?



Sources

This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.

Source: Stack Overflow

Solution Source