C#版采集程序源碼

2019-11-17 04:17:11

字體：大中小

來源：轉載

供稿：網友

因為工作需要，自己寫了一個采集程序，如果冒犯了你的網站，我在這里說一聲對不起！！
哎～！我只是一個普通的程序員　.
namespace CJ
{
    public partial class Form1 : Form
    {
        public int PRoxy = 0;
        public int keyi = 0;
        public int keyj = 0;
        public int keym = 0;
        public int keyn = 0;
        public int sum = 0;

        public string newurl = "";
        public string cururl = "";
        public string dirname = "";
        public string curdir = "";

        public string responseFromServer = "";
        public string filename = "";
        public string sql = "";
        public string mulu = "";

        StringBuilder sbs = new StringBuilder();
        List<Class1> cls = new List<Class1>();
        public ArrayList al = new ArrayList();

        public string insertdl = "insert into mzinedl values(";
        public string insertxl = "insert into mzinexl values(";
        public string insertinfo = "insert into mzineinfo values(";
        public string insertwz = "insert into mzinewz values(";

        public Form1()
        {
            InitializeComponent();
        }

        /// <summary>
        /// 保存網頁
        /// </summary>
        /// <param name="FILE_NAME">文件的路徑</param>
        /// <param name="data">數據</param>
        public void TextToFile(string FILE_NAME, string data)
        {
            if (File.Exists(FILE_NAME))
            {
                return;
            }
            using (StreamWriter sw = File.CreateText(FILE_NAME))
            {
                sw.Write(data);
                sw.Close();
            }
        }
        /// <summary>
        /// 下載文件
        /// </summary>
        /// <param name="PageUrl">網址</param>
        /// <param name="filename">保存文件路徑</param>
        public void DownFile(string PageUrl, string filename)
        {
            if (!Directory.Exists(filename))
            {
                Directory.CreateDirectory(filename);
            }
            string path = PageUrl.Substring(PageUrl.LastIndexOf("/") + 1);
            string dirname = filename + "http://" + path;

            if (File.Exists(dirname))
            {
                return;
            }
            else
            {
                try
                {
                    WebClient wc = new WebClient();
                    WebProxy wp = new WebProxy(al[proxy].ToString(), true);
                    wc.Proxy = wp;
                    wc.DownloadFile(PageUrl, dirname);
                }
                catch (WebException ex)
                {
                    if (ex.Status == WebExceptionStatus.ConnectFailure)
                    {
                        //無法連接到遠程服務器, --換代理 ip
                        //MessageBox.Show(ex.ToString());
                        proxy++;
                        if (proxy >= al.Count)
                        {
                            al = ReadIPproxy("e://test.txt");//初始化代理   IP
                        }
                        DownFile(PageUrl, filename);
                    }
                    else if (ex.Status == WebExceptionStatus.Timeout)
                    {
                        //超時 --換代理 IP
                        //MessageBox.Show(ex.ToString());
                        proxy++;
                        if (proxy >= al.Count)
                        {
                            al = ReadIPproxy("e://test.txt");//初始化代理   IP
                        }
                        DownFile(PageUrl, filename);
                    }
                    else if (ex.Status == WebExceptionStatus.ProtocolError)
                    {
                        //文件未找到--跳出
                        //MessageBox.Show(ex.ToString());
                        return;
                    }
                }
            }

        }
        /// <summary>
        /// 讀文件
        /// </summary>
        /// <param name="FILE_NAME">文件的路徑</param>
        /// <returns>數據</returns>
        public ArrayList ReadIPproxy(string FILE_NAME)
        {
            using (StreamReader sr = File.OpenText(FILE_NAME))
            {
                String input;
                while ((input = sr.ReadLine()) != null)
                {
                    al.Add(input);
                }
                sr.Close();
            }
            return al;

        }
        /// <summary>
        /// 數據庫
        /// </summary>
        public void Executesql()
        {
            SqlHelper.ExecuteNonQuery(SqlHelper.sqlstr, CommandType.Text, sbs.ToString(), null);
        }
        /// <summary>
        /// 讀文件
        /// </summary>
        /// <param name="FILE_NAME">文件的路徑</param>
        /// <returns>數據</returns>
        public string FileToText(string FILE_NAME)
        {
            string data;
            using (StreamReader sr = File.OpenText(FILE_NAME))
            {
                data=sr.ReadToEnd();
                sr.Close();
            }
            return data;
        }
        /// <summary>
        /// 保存SQL
        /// </summary>
        /// <param name="sql"></param>
        public void SaveSqls(string sql)
        {
            sbs.Append(sql).Append("/n");
        }
        /// <summary>
        /// 請求失敗的時候，反復操作
        /// </summary>
        /// <param name="PageUrl"></param>
        /// <returns></returns>
        public string ToServer(string PageUrl)
        {
            string responseFromServer = "";

            try
            {
                while (1 == 1)
                {
                    WebRequest request = WebRequest.Create(PageUrl);
                    WebProxy wp = new WebProxy(al[proxy].ToString(), true);
                    request.Proxy = wp;
                    request.Timeout = 1000 * 60;

                    HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                    Stream dataStream = response.GetResponseStream();
                    StreamReader reader=null;
                    try
                    {
                        reader = new StreamReader(dataStream, System.Text.Encoding.Default);
                        responseFromServer = reader.ReadToEnd();
                    }
                    catch
                    {
                        proxy++;
                        if (proxy >= al.Count)
                        {
                            al = ReadIPproxy("e://test.txt");//初始化代理   IP
                        }
                        ToServer(PageUrl);
                    };
                    reader.Close();
                    dataStream.Close();
                    response.Close();
                    if (responseFromServer.Contains("refresh") || responseFromServer == "")
                    {
                        proxy++;
                        if (proxy >= al.Count)
                        {
                            al = ReadIPproxy("e://test.txt");//初始化代理   IP
                        }
                        //ToServer(PageUrl);
                    }
                    else
                    {
                        break;
                    }
                }
            }
            catch (WebException ex)
            {
                if (ex.Status == WebExceptionStatus.ProtocolError)
                {
                    responseFromServer = "";
                }
                else
                {
                    proxy++;
                    if (proxy >= al.Count)
                    {
                        al = ReadIPproxy("e://test.txt");//初始化代理   IP
                    }
                    ToServer(PageUrl);
                }
            }
            return responseFromServer;
        }
        /// <summary>
        /// 保存xml 文件
        /// </summary>
        public void SaveXmls()
        {
            string pathxml = "";
            foreach (Class1 c in cls)
            {
                Class1 s = c;
                pathxml = s.address;

                if (!File.Exists(pathxml))
                {
                    XmlSerializer xs = new XmlSerializer(typeof(Class1));
                    Stream stream = new FileStream(pathxml, FileMode.Create, File access.Write, FileShare.ReadWrite);
                    xs.Serialize(stream, s);
                    stream.Close();
                }

            }
        }
        /// <summary>
        /// 移除HTMl 標記
        /// </summary>
        /// <param name="Html"></param>
        /// <param name="RegStr"></param>
        /// <returns></returns>
        public static string Remove(string Html)
        {
            //Regex Reg = new Regex(RegStr);
            //foreach (Match m in Reg.Matches(Html))
            //{
            //    Html = Html.Replace(m.Value, "");
            //}
            //return Html.Trim();
            string regesstr = "<.*?>";
            return Regex.Replace(Html, regesstr, string.Empty, RegexOptions.IgnoreCase);
        }
        public static string FilterScript(string content)
        {
            string regexstr = @"<(script)[^>]*>(/s*|.)*<//1>";
            return Regex.Replace(content,regexstr,string.Empty,RegexOptions.IgnoreCase);

        }
        /// <summary>
        /// 過略所有的危險標記
        /// </summary>
        /// <param name="html"></param>
        /// <returns></returns>
        public string wipeScript(string html)
        {
            System.Text.RegularExpressions.Regex regex1 = new System.Text.RegularExpressions.Regex(@"(<script){1,}[^<>]*>[^/0]*(<//script>){1,}", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
            System.Text.RegularExpressions.Regex regex2 = new System.Text.RegularExpressions.Regex(@"href   *=   *[/s/S]*script   *:", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
            System.Text.RegularExpressions.Regex regex3 = new System.Text.RegularExpressions.Regex(@"on[/s/S]*=", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
            System.Text.RegularExpressions.Regex regex4 = new System.Text.RegularExpressions.Regex(@"<iframe[/s/S]+</iframe*>", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
            System.Text.RegularExpressions.Regex regex5 = new System.Text.RegularExpressions.Regex(@"<frameset[/s/S]+</frameset*>", System.Text.RegularExpressions.RegexOptions.IgnoreCase);
            html = regex1.Replace(html, "");   //過濾<script></script>標記
            html = regex2.Replace(html, "");   //過濾href=javascript:   (<A>)   屬性
            html = regex3.Replace(html, "   _disibledevent=");   //過濾其它控件的on...事件
            html = regex4.Replace(html, "");   //過濾iframe
            html = regex5.Replace(html, "");   //過濾frameset
            return html;
        }
        public void HtmlSource(string urlpri)
        {
            //要寫入的文件路徑
            filename = "E://觀2//magazine.html";

            if (!Directory.Exists("E://觀2"))
            {
                Directory.CreateDirectory("E://觀2");
            }
            if (File.Exists(filename))
            {
                responseFromServer=FileToText(filename); //存在

            }
            else
            {
                responseFromServer = ToServer(urlpri); //不存在

            }
            sum++;
            if (responseFromServer != "")
            {
                //分析內容
                TextToFile(filename,responseFromServer);

                MatchCollection mc = Regex.Matches(responseFromServer, @"href=""/magazine/(.*)""><b>(.*)</b>", RegexOptions.IgnoreCase);
                foreach (Match m in mc)
                {
                    newurl = m.Groups[1].Value;
                    dirname = m.Groups[2].Value;

                    int key = ++keyi;
                    sql = insertdl + key + ",'" + dirname + "')";
                    SaveSqls(sql);

cururl = urlpri + newurl;
curdir = "E://觀2//" + dirname;

                    one(cururl, curdir,key);
                }
                SaveXmls();
                Executesql();

                this.textBox1.Text = sum.ToString();
                MessageBox.Show("采集成功!");
            }
        }
        public void one(string urlpri,string _dirname,int _key)
        {
            //要寫入的文件路徑
            filename = _dirname +"http://"+ urlpri.Substring(urlpri.LastIndexOf("/") + 1);

            if (!Directory.Exists(_dirname))
            {
                Directory.CreateDirectory(_dirname);
            }
            if (File.Exists(filename))
            {
                responseFromServer = FileToText(filename);
            }
            else
            {
                responseFromServer = ToServer(urlpri);
            }
            sum++;
            if (responseFromServer != "")
            {
                TextToFile(filename, responseFromServer);

MatchCollection mc = Regex.Matches(responseFromServer, @"href=""/././(.*list.html)""[/s/S]*?《(.*?)》", RegexOptions.IgnoreCase);

                foreach (Match m in mc)
                {
                    newurl = m.Groups[1].Value;
                    dirname = m.Groups[2].Value;

cururl = " curdir = _dirname + "http://" + dirname;

two(cururl, curdir, _key);

                }
            }
        }
        public void two(string urlpri,string _dirname,int _key)
        {
            filename = urlpri.Substring(0, urlpri.LastIndexOf("/"));
            filename = filename.Substring(filename.LastIndexOf("/") + 1) + ".html";
            filename = _dirname + "http://" + filename;

            if (!Directory.Exists(_dirname))
            {
                Directory.CreateDirectory(_dirname);
            }

            if (File.Exists(filename))
            {
                responseFromServer = FileToText(filename);
            }
            else
            {
                responseFromServer = ToServer(urlpri);
            }
            sum++;
            if (responseFromServer != "")
            {
                TextToFile(filename, responseFromServer);