Data conversion program from Mysql to Mongodb _ MySQL

Source: Internet
Author: User
The previously written data conversion program from Mysql to Mongodb has been turned over, and is sufficient in the case of a small amount of data. -*-Coding: UTF-8-*-importsys, osimportmultiprocessingimportloggingimportrandomimporttime. the previously written Mysql-to-Mongodb data conversion program is flipped out, which is sufficient when the data volume is small.
# -*- coding: utf-8 -*-import sys, osimport multiprocessingimport loggingimport randomimport time, datetimeimport MySQLdbfrom MySQLdb import cursorsfrom pymongo import MongoClientclass Config:    tables = ['hs_card', 'hs_hero', 'hs_set', 'hs_skill', 'hs_level', 'hs_pack', 'hs_salesevent']    #tables = ['hs_card', 'hs_hero', 'hs_set', 'hs_skill', 'hs_level']    index = {             'hs_card': ['name'],    }class Mysql2Mongo(object):    mysql_host = '50.23.4.2'    mysql_port = 3306    mysql_user = "root"    mysql_pass = "stm123"    mysql_db = "ccg_alpha"    mongo_host = '50.23.4.2'    mongo_port = 27017    conn = None    cursor = None    mongo = None    mongodb = None    def __init__(self, logger):self.logger = loggerself.conn = self.getMysqlConn()self.cursor = self.conn.cursor()self.mongo = MongoClient(host=self.mongo_host, port=self.mongo_port)self.mongodb = self.mongo['ccg_alpha']    def getMysqlConn(self):        return MySQLdb.connect(host=self.mysql_host, port=self.mysql_port, user=self.mysql_user, \                 passwd=self.mysql_pass, db=self.mysql_db, cursorclass=MySQLdb.cursors.SSCursor)    def setMongoCollectionDocument(self, table, data):if(isinstance(data, dict) == False):return Falseelse:self.mongodb[table].insert(data)    def getMysqlTableDesc(self, table):sql = """desc %s""" % (table)n = self.cursor.execute(sql)data = self.cursor.fetchall()keys = []types = []for row in data:key = str(row[0])if(row[1].find('int') >= 0):type = 1elif (row[1].find('char') >= 0):type = 2elif (row[1].find('text') >= 0):type = 2elif(row[1].find('decimal') >= 0):type = 3else:type = 2keys.append(key)types.append(type)return keys, types    def mysql2Mongo(self, table):        self.mongodb[table].drop()        keys, types = self.getMysqlTableDesc(table)        sql = """select * from  %s order by id asc""" % (table)        n = self.cursor.execute(sql)        data = self.cursor.fetchall()        #print table, keys, types        for row in data:ret = {}for k, key in enumerate(keys):if key == 'id':key = '_id'#ret[key] = int(row[k])if(types[k] == 1):if row[k]==None:ret[key]= 0continue#print k, key, rowret[key] = int(row[k])elif(types[k] == 2):if row[k]==None:ret[key]= ''continueret[key] = str(row[k])elif(types[k] == 3):if row[k]==None:ret[key]= ''continueret[key] = float(row[k])else:if row[k]==None:ret[key]= ''continueret[key] = str(row[k])#if(table== 'hs_card') or (table== 'hs_hero'):#ret['rand'] = random.random()print retself.setMongoCollectionDocument(table, ret)    def __del__(self):self.mongo.close()self.cursor.close()self.conn.close()if __name__ == '__main__':    multiprocessing.log_to_stderr()    logger = multiprocessing.get_logger()    logger.setLevel(logging.INFO)    # args = sys.argv    t1 = time.time()    cls = Mysql2Mongo(logger)    for tb in Config.tables:cls.mysql2Mongo(tb)    #index        for t, f in Config.index.items():        pass        print time.time() - t1    logger.info("done")
Related Article

Contact Us

The content source of this page is from Internet, which doesn't represent Alibaba Cloud's opinion; products and services mentioned on that page don't have any relationship with Alibaba Cloud. If the content of the page makes you feel confusing, please write us an email, we will handle the problem within 5 days after receiving your email.

If you find any instances of plagiarism from the community, please send an email to: info-contact@alibabacloud.com and provide relevant evidence. A staff member will contact you within 5 working days.

A Free Trial That Lets You Build Big!

Start building with 50+ products and up to 12 months usage for Elastic Compute Service

  • Sales Support

    1 on 1 presale consultation

  • After-Sales Support

    24/7 Technical Support 6 Free Tickets per Quarter Faster Response

  • Alibaba Cloud offers highly flexible support services tailored to meet your exact needs.