說明:1)需要安裝擴展庫openpyxl;2)隨著數據庫的增大,導入速度可能會有所下降;3)本文只考慮Python代碼優化,沒有涉及數據庫的優化;4)本文要點在于使用executemany實現批量數據導入,通過減少事務提交次數提高導入速度。
from random import choice, randrangefrom string import digits, ascii_lettersfrom os import listdirimport sqlite3from time import timefrom openpyxl import Workbook, load_workbook
def generateRandomData(): #total表示記錄總條數 global total characters = digits+ascii_letters for i in range(50): xlsName = 'xlsxs//'+str(i)+'.xlsx' #隨機數,每個xlsx文件的行數不一樣 totalLines = randrange(10**5) wb = Workbook() ws = wb.worksheets[0] #表頭 ws.append(['a', 'b', 'c', 'd', 'e']) #隨機數據,每行5個字段,每個字段30個字符 for j in range(totalLines): line = [''.join((choice(characters)for ii in range(30))) for jj in range(5)] ws.append(line) total += 1 #保存xlsx文件 wb.save(xlsName)
#針對每個xlsx文件的生成器def eachXlsx(xlsxFn): wb = load_workbook(xlsxFn) ws = wb.worksheets[0] for index, row in enumerate(ws.rows): #忽略表頭 if index == 0: continue yield tuple(map(lambda x:x.value, row))
#導入def xlsx2sqlite(): #獲取所有xlsx文件 xlsxs = ('xlsxs//'+fn for fn in listdir('xlsxs')) #連接數據庫,創建游標 conn = sqlite3.connect('data.db') cur = conn.cursor() for xlsx in xlsxs: #批量導入,減少提交事務的次數,可以提高速度 sql = 'insert into fromxlsx values(?,?,?,?,?)' cur.executemany(sql, eachXlsx(xlsx)) conn.commit()
total = 0
generateRandomData()
start = time()xlsx2sqlite()delta = time()-start 運行結果: 導入用時: 326.4754948616028導入速度(條/秒): 7105.5317673486825
新聞熱點
疑難解答