本文實例講述了Python實現讀取TXT文件數據并存進內置數據庫SQLite3的方法。分享給大家供大家參考,具體如下:
當TXT文件太大,計算機內存不夠時,我們可以選擇按行讀取TXT文件,并將其存儲進Python內置輕量級splite數據庫,這樣可以加快數據的讀取速度,當我們需要重復讀取數據時,這樣的速度加快所帶來的時間節省是非常可觀的,比如,當我們在訓練數據時,要迭代10萬次,即要從文件中讀取10萬次,即使每次只加快0.1秒,那么也能節省幾個小時的時間了。
#創建數據庫并把txt文件的數據存進數據庫import sqlite3 #導入sqlite3cx = sqlite3.connect('./train.db') #創建數據庫,如果數據庫已經存在,則鏈接數據庫;如果數據庫不存在,則先創建數據庫,再鏈接該數據庫。cu = cx.cursor() #定義一個游標,以便獲得查詢對象。cu.execute('create table if not exists train4 (id integer primary key,name text)') #創建表fr = open('data_sample.txt') #打開要讀取的txt文件i = 0for line in fr.readlines(): #將數據按行插入數據庫的表train4中。 cu.execute('insert into train4 values(?,?)',(i,line)) i +=1cu.close() #關閉游標cx.commit() #事務提交cx.close() #關閉數據庫查詢數據:
cu.execute('select * from train4 where id = ?',(i,)) #i代表你要讀取表train4中某一行的數據result = cu.fetchall()注:如果前面已經關閉了數據庫,那么在查詢時要重新打開數據庫,并創建游標。這一點要注意一下。
完整的查詢程序是這樣的:
import sqlite3cx = sqlite3.connect('./train.db')cu = cx.cursor()for i in range(5): cu.execute('select * from train4 where id = ?',(i,)) result = cu.fetchall() cx.commit()cu.close()cx.close()另:這里再為大家附帶一個SQLite3數據操作類供大家參考使用:
import sqlite3# ***************************************************# *# * Description: Python操作SQLite3數據庫輔助類(查詢構造器)# * Author: wangye# *# ***************************************************def _wrap_value(value): return repr(value)def _wrap_values(values): return list(map(_wrap_value, values))def _wrap_fields(fields): for key,value in fields.items(): fields[key] = _wrap_value(value) return fieldsdef _concat_keys(keys): return "[" + "],[".join(keys) + "]"def _concat_values(values): return ",".join(values)def _concat_fields(fields, operator = (None, ",")): if operator: unit_operator, group_operator = operator # fields = _wrap_fields(fields) compiled = [] for key,value in fields.items(): compiled.append("[" + key + "]") if unit_operator: compiled.append(unit_operator) compiled.append(value) compiled.append(group_operator) compiled.pop() # pop last group_operator return " ".join(compiled)class DataCondition(object): """ 本類用于操作SQL構造器輔助類的條件語句部分 例如: DataCondition(("=", "AND"), id = 26) DataCondition(("=", "AND"), True, id = 26) """ def __init__(self, operator = ("=", "AND"), ingroup = True, **kwargs): """ 構造方法 參數: operator 操作符,分為(表達式操作符, 條件運算符) ingroup 是否分組,如果分組,將以括號包含 kwargs 鍵值元組,包含數據庫表的列名以及值 注意這里的等于號不等于實際生成SQL語句符號 實際符號是由operator[0]控制的 例如: DataCondition(("=", "AND"), id = 26) (id=26) DataCondition((">", "OR"), id = 26, age = 35) (id>26 OR age>35) DataCondition(("LIKE", "OR"), False, name = "John", company = "Google") name LIKE 'John' OR company LIKE "Google" """ self.ingroup = ingroup self.fields = kwargs self.operator = operator def __unicode__(self): self.fields = _wrap_fields(self.fields) result = _concat_fields(self.fields, self.operator) if self.ingroup: return "(" + result + ")" return result def __str__(self): return self.__unicode__() def toString(self): return self.__unicode__()class DataHelper(object): """ SQLite3 數據查詢輔助類 """ def __init__(self, filename): """ 構造方法 參數: filename 為SQLite3 數據庫文件名 """ self.file_name = filename def open(self): """ 打開數據庫并設置游標 """ self.connection = sqlite3.connect(self.file_name) self.cursor = self.connection.cursor() return self def close(self): """ 關閉數據庫,注意若不顯式調用此方法, 在類被回收時也會嘗試調用 """ if hasattr(self, "connection") and self.connection: self.connection.close() def __del__(self): """ 析構方法,做一些清理工作 """ self.close() def commit(self): """ 提交事務 SELECT語句不需要此操作,默認的execute方法的 commit_at_once設為True會隱式調用此方法, 否則就需要顯示調用本方法。 """ self.connection.commit() def execute(self, sql = None, commit_at_once = True): """ 執行SQL語句 參數: sql 要執行的SQL語句,若為None,則調用構造器生成的SQL語句。 commit_at_once 是否立即提交事務,如果不立即提交, 對于非查詢操作,則需要調用commit顯式提交。 """ if not sql: sql = self.sql self.cursor.execute(sql) if commit_at_once: self.commit() def fetchone(self, sql = None): """ 取一條記錄 """ self.execute(sql, False) return self.cursor.fetchone() def fetchall(self, sql = None): """ 取所有記錄 """ self.execute(sql, False) return self.cursor.fetchall() def __concat_keys(self, keys): return _concat_keys(keys) def __concat_values(self, values): return _concat_values(values) def table(self, *args): """ 設置查詢的表,多個表名用逗號分隔 """ self.tables = args self.tables_snippet = self.__concat_keys(self.tables) return self def __wrap_value(self, value): return _wrap_value(value) def __wrap_values(self, values): return _wrap_values(values) def __wrap_fields(self, fields): return _wrap_fields(fields) def __where(self): # self.condition_snippet if hasattr(self, "condition_snippet"): self.where_snippet = " WHERE " + self.condition_snippet def __select(self): template = "SELECT %(keys)s FROM %(tables)s" body_snippet_fields = { "tables" : self.tables_snippet, "keys" : self.__concat_keys(self.body_keys), } self.sql = template % body_snippet_fields def __insert(self): template = "INSERT INTO %(tables)s (%(keys)s) VALUES (%(values)s)" body_snippet_fields = { "tables" : self.tables_snippet, "keys" : self.__concat_keys(list(self.body_fields.keys())), "values" : self.__concat_values(list(self.body_fields.values())) } self.sql = template % body_snippet_fields def __update(self): template = "UPDATE %(tables)s SET %(fields)s" body_snippet_fields = { "tables" : self.tables_snippet, "fields" : _concat_fields(self.body_fields, ("=",",")) } self.sql = template % body_snippet_fields def __delete(self): template = "DELETE FROM %(tables)s" body_snippet_fields = { "tables" : self.tables_snippet } self.sql = template % body_snippet_fields def __build(self): { "SELECT": self.__select, "INSERT": self.__insert, "UPDATE": self.__update, "DELETE": self.__delete }[self.current_token]() def __unicode__(self): return self.sql def __str__(self): return self.__unicode__() def select(self, *args): self.current_token = "SELECT" self.body_keys = args self.__build() return self def insert(self, **kwargs): self.current_token = "INSERT" self.body_fields = self.__wrap_fields(kwargs) self.__build() return self def update(self, **kwargs): self.current_token = "UPDATE" self.body_fields = self.__wrap_fields(kwargs) self.__build() return self def delete(self, *conditions): self.current_token = "DELETE" self.__build() #if *conditions: self.where(*conditions) return self def where(self, *conditions): conditions = list(map(str, conditions)) self.condition_snippet = " AND ".join(conditions) self.__where() if hasattr(self, "where_snippet"): self.sql += self.where_snippet return self
希望本文所述對大家Python程序設計有所幫助。
新聞熱點
疑難解答