/** Author:Sunjoy at CCNU* 如果您改進(jìn)了這個(gè)類請(qǐng)發(fā)一份代碼給我(ccnusjy 在gmail.com)*/using System;using System.Data;using System.Configuration;using System.Net;using System.IO;using System.Text;using System.Collections.Generic;using System.Text.RegularExpressions;using System.Threading;using System.Web;/// <summary>/// 網(wǎng)頁類/// </summary>public class WebPage{ #region 私有成員 private Uri m_uri; //網(wǎng)址 private List<Link> m_links; //此網(wǎng)頁上的鏈接 private string m_title;  //此網(wǎng)頁的標(biāo)題 private string m_html;   //此網(wǎng)頁的HTML代碼 private string m_outstr;  //此網(wǎng)頁可輸出的純文本 private bool m_good;   //此網(wǎng)頁是否可用 private int m_pagesize;  //此網(wǎng)頁的大小 private static Dictionary<string, CookieContainer> webcookies = new Dictionary<string, CookieContainer>();//存放所有網(wǎng)頁的Cookie private string m_post; //此網(wǎng)頁的登陸頁需要的POST數(shù)據(jù) private string m_loginurl; //此網(wǎng)頁的登陸頁 #endregion #region 私有方法 /// <summary> /// 這私有方法從網(wǎng)頁的HTML代碼中分析出鏈接信息 /// </summary> /// <returns>List<Link></returns> private List<Link> getLinks() {  if (m_links.Count == 0)  {   Regex[] regex = new Regex[2];   regex[0] = new Regex("(?m)<a[^><]+href=(/"|')?(?<url>([^>/"'//s)])+)(/"|')?[^>]*>(?<text>(//w|//W)*?)</", RegexOptions.Multiline | RegexOptions.IgnoreCase);   regex[1] = new Regex("<[i]*frame[^><]+src=(/"|')?(?<url>([^>/"'//s)])+)(/"|')?[^>]*>", RegexOptions.Multiline | RegexOptions.IgnoreCase);   for (int i = 0; i < 2; i++)   {    Match match = regex[i].Match(m_html);    while (match.Success)    {     try     {      string url = new Uri(m_uri, match.Groups["url"].Value).AbsoluteUri;      string text = "";      if (i == 0) text = new Regex("(<[^>]+>)|(//s)|( )|&|/"", RegexOptions.Multiline | RegexOptions.IgnoreCase).Replace(match.Groups["text"].Value, "");      Link link = new Link(url, text);      m_links.Add(link);     }     catch(Exception ex){Console.WriteLine(ex.Message); };     match = match.NextMatch();    }   }  }  return m_links; } /// <summary> /// 此私有方法從一段HTML文本中提取出一定字?jǐn)?shù)的純文本 /// </summary> /// <param name="instr">HTML代碼</param> /// <param name="firstN">提取從頭數(shù)多少個(gè)字</param> /// <param name="withLink">是否要鏈接里面的字</param> /// <returns>純文本</returns> private string getFirstNchar(string instr, int firstN, bool withLink) {  if (m_outstr == "")  {   m_outstr = instr.Clone() as string;   m_outstr = new Regex(@"(?m)<script[^>]*>(/w|/W)*?</script[^>]*>", RegexOptions.Multiline | RegexOptions.IgnoreCase ).Replace(m_outstr, "");   m_outstr = new Regex(@"(?m)<style[^>]*>(/w|/W)*?</style[^>]*>", RegexOptions.Multiline | RegexOptions.IgnoreCase ).Replace(m_outstr, "");   m_outstr = new Regex(@"(?m)<select[^>]*>(/w|/W)*?</select[^>]*>", RegexOptions.Multiline | RegexOptions.IgnoreCase ).Replace(m_outstr, "");   if (!withLink) m_outstr = new Regex(@"(?m)<a[^>]*>(/w|/W)*?</a[^>]*>", RegexOptions.Multiline | RegexOptions.IgnoreCase).Replace(m_outstr, "");   Regex objReg = new System.Text.RegularExpressions.Regex("(<[^>]+?>)| ", RegexOptions.Multiline | RegexOptions.IgnoreCase);   m_outstr = objReg.Replace(m_outstr, "");   Regex objReg2 = new System.Text.RegularExpressions.Regex("(//s)+", RegexOptions.Multiline | RegexOptions.IgnoreCase);   m_outstr = objReg2.Replace(m_outstr, " ");  }  return m_outstr.Length > firstN ? m_outstr.Substring(0, firstN) : m_outstr; } /// <summary> /// 此私有方法返回一個(gè)IP地址對(duì)應(yīng)的無符號(hào)整數(shù) /// </summary> /// <param name="x">IP地址</param> /// <returns></returns> private uint getuintFromIP(IPAddress x) {  Byte[] bt = x.GetAddressBytes();  uint i = (uint)(bt[0] * 256 * 256 * 256);  i += (uint)(bt[1] * 256 * 256);  i += (uint)(bt[2] * 256);  i += (uint)(bt[3]);  return i; } #endregion #region 公有文法 /// <summary> /// 此公有方法提取網(wǎng)頁中一定字?jǐn)?shù)的純文本，包括鏈接文字 /// </summary> /// <param name="firstN">字?jǐn)?shù)</param> /// <returns></returns> public string getContext(int firstN) {  return getFirstNchar(m_html, firstN, true); } /// <summary> /// 此公有方法提取網(wǎng)頁中一定字?jǐn)?shù)的純文本，不包括鏈接文字 /// </summary> /// <param name="firstN"></param> /// <returns></returns> public string getContextWithOutLink(int firstN) {  return getFirstNchar(m_html, firstN, false); } /// <summary> /// 此公有方法從本網(wǎng)頁的鏈接中提取一定數(shù)量的鏈接，該鏈接的URL滿足某正則式 /// </summary> /// <param name="pattern">正則式</param> /// <param name="count">返回的鏈接的個(gè)數(shù)</param> /// <returns>List<Link></returns> public List<Link> getSpecialLinksByUrl(string pattern,int count) {  if(m_links.Count==0)getLinks();  List<Link> SpecialLinks = new List<Link>();  List<Link>.Enumerator i;  i = m_links.GetEnumerator();  int cnt = 0;  while (i.MoveNext() && cnt<count)  {   if (new Regex(pattern, RegexOptions.Multiline | RegexOptions.IgnoreCase ).Match(i.Current.url).Success)   {    SpecialLinks.Add(i.Current);    cnt++;   }  }   return SpecialLinks; } /// <summary> /// 此公有方法從本網(wǎng)頁的鏈接中提取一定數(shù)量的鏈接，該鏈接的文字滿足某正則式 /// </summary> /// <param name="pattern">正則式</param> /// <param name="count">返回的鏈接的個(gè)數(shù)</param> /// <returns>List<Link></returns> public List<Link> getSpecialLinksByText(string pattern,int count) {  if (m_links.Count == 0) getLinks();  List<Link> SpecialLinks = new List<Link>();  List<Link>.Enumerator i;  i = m_links.GetEnumerator();  int cnt = 0;  while (i.MoveNext() && cnt < count)  {   if (new Regex(pattern, RegexOptions.Multiline | RegexOptions.IgnoreCase ).Match(i.Current.text).Success)   {    SpecialLinks.Add(i.Current);    cnt++;   }  }  return SpecialLinks; } /// <summary> /// 此公有方法獲得所有鏈接中在一定IP范圍的鏈接 /// </summary> /// <param name="_ip_start">起始IP</param> /// <param name="_ip_end">終止IP</param> /// <returns></returns> public List<Link> getSpecialLinksByIP(string _ip_start, string _ip_end) {  IPAddress ip_start = IPAddress.Parse(_ip_start);  IPAddress ip_end = IPAddress.Parse(_ip_end);  if (m_links.Count == 0) getLinks();  List<Link> SpecialLinks = new List<Link>();  List<Link>.Enumerator i;  i = m_links.GetEnumerator();  while (i.MoveNext())  {   IPAddress ip;   try   {    ip = Dns.GetHostEntry(new Uri(i.Current.url).Host).AddressList[0];   }   catch { continue; }   if(getuintFromIP(ip)>=getuintFromIP(ip_start) && getuintFromIP(ip)<=getuintFromIP(ip_end))   {    SpecialLinks.Add(i.Current);   }  }  return SpecialLinks; } /// <summary> /// 這公有方法提取本網(wǎng)頁的純文本中滿足某正則式的文字 /// </summary> /// <param name="pattern">正則式</param> /// <returns>返回文字</returns> public string getSpecialWords(string pattern) {  if (m_outstr == "") getContext(Int16.MaxValue);  Regex regex = new Regex(pattern, RegexOptions.Multiline | RegexOptions.IgnoreCase );  Match mc=regex.Match(m_outstr);  if (mc.Success)   return mc.Groups[1].Value;  return string.Empty; } #endregion #region 構(gòu)造函數(shù) private void Init(string _url) {  try  {   m_uri = new Uri(_url);   m_links = new List<Link>();   m_html = "";   m_outstr = "";   m_title = "";   m_good = true;   if (_url.EndsWith(".rar") || _url.EndsWith(".dat") || _url.EndsWith(".msi"))   {    m_good = false;    return;   }   HttpWebRequest rqst = (HttpWebRequest)WebRequest.Create(m_uri);   rqst.AllowAutoRedirect = true;   rqst.MaximumAutomaticRedirections = 3;   rqst.UserAgent = "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)";   rqst.KeepAlive = true;   rqst.Timeout = 30000;   lock (WebPage.webcookies)   {    if (WebPage.webcookies.ContainsKey(m_uri.Host))     rqst.CookieContainer = WebPage.webcookies[m_uri.Host];    else    {     CookieContainer cc = new CookieContainer();     WebPage.webcookies[m_uri.Host] = cc;     rqst.CookieContainer = cc;    }   }   HttpWebResponse rsps = (HttpWebResponse)rqst.GetResponse();   Stream sm = rsps.GetResponseStream();   if (!rsps.ContentType.ToLower().StartsWith("text/") || rsps.ContentLength > 1 << 22)   {    rsps.Close();    m_good = false;    return;   }   Encoding cding = System.Text.Encoding.Default;   string contenttype=rsps.ContentType.ToLower();   int ix = contenttype.IndexOf("charset=");   if (ix != -1)   {    try    {     cding = System.Text.Encoding.GetEncoding(rsps.ContentType.Substring(ix + "charset".Length + 1));    }    catch    {     cding = Encoding.Default;    }    m_html = new StreamReader(sm, cding).ReadToEnd();   }   else   {    m_html = new StreamReader(sm, cding).ReadToEnd();    Regex regex = new Regex("charset=(?<cding>[^=]+)?/"",RegexOptions.IgnoreCase);    string strcding = regex.Match(m_html).Groups["cding"].Value;    try    {     cding = Encoding.GetEncoding(strcding);    }    catch{     cding = Encoding.Default;    }    byte[] bytes=Encoding.Default.GetBytes(m_html.ToCharArray());    m_html = cding.GetString(bytes);    if (m_html.Split('?').Length > 100)    {     m_html=Encoding.Default.GetString(bytes);    }   }      m_pagesize = m_html.Length;   m_uri = rsps.ResponseUri;   rsps.Close();  }  catch (Exception ex)  {   Console.WriteLine(ex.Message+m_uri.ToString());   m_good = false;  } } public WebPage(string _url) {  string uurl = "";  try  {   uurl = Uri.UnescapeDataString(_url);   _url = uurl;  }  catch { };  Regex re = new Regex("(?<h>[^/x00-/xff]+)");  Match mc = re.Match(_url);  if (mc.Success)  {   string han = mc.Groups["h"].Value;   _url = _url.Replace(han, System.Web.HttpUtility.UrlEncode(han, Encoding.GetEncoding("GB2312")));  }  Init(_url); } public WebPage(string _url, string _loginurl, string _post) {  string uurl = "";  try  {   uurl = Uri.UnescapeDataString(_url);   _url = uurl;  }  catch { };  Regex re = new Regex("(?<h>[^/x00-/xff]+)");  Match mc = re.Match(_url);  if (mc.Success)  {   string han = mc.Groups["h"].Value;   _url = _url.Replace(han, System.Web.HttpUtility.UrlEncode(han, Encoding.GetEncoding("GB2312")));  }  if (_loginurl.Trim() == "" || _post.Trim() == "" || WebPage.webcookies.ContainsKey(new Uri(_url).Host))  {   Init(_url);  }  else  {   #region 登陸   string indata = _post;   m_post = _post;   m_loginurl = _loginurl;   byte[] bytes = Encoding.Default.GetBytes(_post);   CookieContainer myCookieContainer = new CookieContainer();   try   {    //新建一個(gè)CookieContainer來存放Cookie集合     HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(_loginurl);    //新建一個(gè)HttpWebRequest     myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";    myHttpWebRequest.AllowAutoRedirect = false;    myHttpWebRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)";    myHttpWebRequest.Timeout = 60000;    myHttpWebRequest.KeepAlive = true;    myHttpWebRequest.ContentLength = bytes.Length;    myHttpWebRequest.Method = "POST";    myHttpWebRequest.CookieContainer = myCookieContainer;    //設(shè)置HttpWebRequest的CookieContainer為剛才建立的那個(gè)myCookieContainer     Stream myRequestStream = myHttpWebRequest.GetRequestStream();    myRequestStream.Write(bytes, 0, bytes.Length);    myRequestStream.Close();    HttpWebResponse myHttpWebResponse = (HttpWebResponse)myHttpWebRequest.GetResponse();    foreach (Cookie ck in myHttpWebResponse.Cookies)    {     myCookieContainer.Add(ck);    }    myHttpWebResponse.Close();   }   catch   {    Init(_url);    return;   }   #endregion   #region 登陸后再訪問頁面   try   {    m_uri = new Uri(_url);    m_links = new List<Link>();    m_html = "";    m_outstr = "";    m_title = "";    m_good = true;    if (_url.EndsWith(".rar") || _url.EndsWith(".dat") || _url.EndsWith(".msi"))    {     m_good = false;     return;    }    HttpWebRequest rqst = (HttpWebRequest)WebRequest.Create(m_uri);    rqst.AllowAutoRedirect = true;    rqst.MaximumAutomaticRedirections = 3;    rqst.UserAgent = "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)";    rqst.KeepAlive = true;    rqst.Timeout = 30000;    rqst.CookieContainer = myCookieContainer;    lock (WebPage.webcookies)    {     WebPage.webcookies[m_uri.Host] = myCookieContainer;    }    HttpWebResponse rsps = (HttpWebResponse)rqst.GetResponse();    Stream sm = rsps.GetResponseStream();    if (!rsps.ContentType.ToLower().StartsWith("text/") || rsps.ContentLength > 1 << 22)    {     rsps.Close();     m_good = false;     return;    }    Encoding cding = System.Text.Encoding.Default;    int ix = rsps.ContentType.ToLower().IndexOf("charset=");    if (ix != -1)    {     try     {      cding = System.Text.Encoding.GetEncoding(rsps.ContentType.Substring(ix + "charset".Length + 1));     }     catch     {      cding = Encoding.Default;     }    }    m_html = new StreamReader(sm, cding).ReadToEnd();    m_pagesize = m_html.Length;    m_uri = rsps.ResponseUri;    rsps.Close();   }   catch (Exception ex)   {    Console.WriteLine(ex.Message+m_uri.ToString());    m_good = false;   }   #endregion  } } #endregion #region 屬性 /// <summary> /// 通過此屬性可獲得本網(wǎng)頁的網(wǎng)址，只讀 /// </summary> public string URL {  get  {   return m_uri.AbsoluteUri;  } } /// <summary> /// 通過此屬性可獲得本網(wǎng)頁的標(biāo)題，只讀 /// </summary> public string Title {  get  {   if (m_title == "")   {    Regex reg = new Regex(@"(?m)<title[^>]*>(?<title>(?:/w|/W)*?)</title[^>]*>", RegexOptions.Multiline | RegexOptions.IgnoreCase );    Match mc = reg.Match(m_html);    if (mc.Success)     m_title= mc.Groups["title"].Value.Trim();   }   return m_title;  } } /// <summary> /// 此屬性獲得本網(wǎng)頁的所有鏈接信息，只讀 /// </summary> public List<Link> Links {  get  {   if (m_links.Count == 0) getLinks();   return m_links;  } } /// <summary> /// 此屬性返回本網(wǎng)頁的全部純文本信息，只讀 /// </summary> public string Context {  get  {   if (m_outstr == "") getContext(Int16.MaxValue);   return m_outstr;  } } /// <summary> /// 此屬性獲得本網(wǎng)頁的大小 /// </summary> public int PageSize {  get  {   return m_pagesize;  } } /// <summary> /// 此屬性獲得本網(wǎng)頁的所有站內(nèi)鏈接 /// </summary> public List<Link> InsiteLinks {  get  {   return getSpecialLinksByUrl("^http://"+m_uri.Host,Int16.MaxValue);  } } /// <summary> /// 此屬性表示本網(wǎng)頁是否可用 /// </summary> public bool IsGood {  get  {   return m_good;  } } /// <summary> /// 此屬性表示網(wǎng)頁的所在的網(wǎng)站 /// </summary> public string Host {  get  {   return m_uri.Host;  } } /// <summary> /// 此網(wǎng)頁的登陸頁所需的POST數(shù)據(jù) /// </summary> public string PostStr {  get  {   return m_post;  } } /// <summary> /// 此網(wǎng)頁的登陸頁 /// </summary> public string LoginURL {  get  {   return m_loginurl;  } } #endregion}/// <summary>/// 鏈接類/// </summary>public class Link{ public string url; //鏈接網(wǎng)址 public string text; //鏈接文字 public Link(string _url, string _text) {  url = _url;  text = _text; }}