package cn.jado.ctt_check.test;
import java.io.BufferedReader; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.List;
import org.apache.log4j.Logger;
import cn.jado.ctt_check.util.IOUtil;
public class SpiltFile { PRivate static final Logger logger = Logger.getLogger(SpiltFile.class);
public static void main(String[] args) { String path = "F:/usr/local/統計/utf-8_20170221_wechat_article.DATA"; readFileByLineToList(path, "utf-8");}/*** * 將一個大文件以1百萬一個文件進行拆分 * @author jado * @param fileName * @param charsetName * @return */public static List<String> readFileByLineToList(String fileName,String charsetName){ List<String> lineList = new ArrayList<String>(); String path = IOUtil.PathParser(fileName)[0]+"/wechat_msg/"; IOUtil.mkDirs(path); if(charsetName == null){ charsetName = System.getProperty("file.encoding"); } BufferedReader br = null; InputStreamReader isReader = null; try { if(charsetName == null){ charsetName = System.getProperty("file.encoding"); } isReader = new InputStreamReader(new FileInputStream(fileName),charsetName); br = new BufferedReader(isReader); String tempString = null; int v = 0; while((tempString = br.readLine())!= null){ v+=1; // 百萬條數據分成一個文件 lineList.add(tempString); if(v%100000==0){ for(String line :lineList){ IOUtil.writeFile(path+v+"_wechat_article.DATA", line+"/n", "utf-8",true); } lineList.clear(); } } } catch (UnsupportedEncodingException e1) { // TODO Auto-generated catch block logger.error("解析文件編碼異常",e1); } catch (FileNotFoundException e2) { // TODO Auto-generated catch block logger.error("文件沒有找到異常",e2); } catch (IOException e3) { // TODO Auto-generated catch block logger.error("Io操作異常",e3); }finally{ if(br != null){ try { br.close(); br = null; } catch (IOException e4) { // TODO Auto-generated catch block logger.error("文件關閉異常",e4); } } if(isReader != null){ try { isReader.close(); } catch (IOException e5) { // TODO Auto-generated catch block logger.error("文件關閉異常",e5); } } } return lineList;}}
新聞熱點
疑難解答
