using System;using System.Collections.Concurrent;using System.Collections.Generic;using System.ComponentModel;using System.Data;using System.Drawing;using System.IO;using System.Linq;using System.Net;using System.Text;using System.Text.RegularExpressions;using System.Threading;using System.Threading.Tasks;using System.Windows.Forms;using Common.Utils;using Utils;namespace 爬蟲{  public partial class Form1 : Form  {    #region 變量    /// <summary>    /// 已完成字節數    /// </summary>    private long completedCount = 0;    /// <summary>    /// 是否完成    /// </summary>    private bool isCompleted = true;    /// <summary>    /// 數據塊隊列    /// </summary>    private ConcurrentQueue<MemoryStream> msQueue = new ConcurrentQueue<MemoryStream>();    /// <summary>    /// 下載開始位置    /// </summary>    private long range = 0;    /// <summary>    /// 文件大小    /// </summary>    private long total = 0;    /// <summary>    /// 一段時間內的完成節點數，計算網速用    /// </summary>    private long unitCount = 0;    /// <summary>    /// 上次計時時間，計算網速用    /// </summary>    private DateTime lastTime = DateTime.MinValue;    /// <summary>    /// 一段時間內的完成字節數，控制網速用    /// </summary>    private long unitCountForLimit = 0;    /// <summary>    /// 上次計時時間，控制網速用    /// </summary>    private DateTime lastTimeForLimit = DateTime.MinValue;    /// <summary>    /// 下載文件sleep時間，控制速度用    /// </summary>    private int sleepTime = 1;    #endregion    #region Form1    public Form1()    {      InitializeComponent();    }    #endregion    #region Form1_Load    private void Form1_Load(object sender, EventArgs e)    {      lblMsg.Text = string.Empty;      lblByteMsg.Text = string.Empty;      lblSpeed.Text = string.Empty;    }    #endregion    #region Form1_FormClosing    private void Form1_FormClosing(object sender, FormClosingEventArgs e)    {    }    #endregion    #region btnDownload_Click 下載    private void btnDownload_Click(object sender, EventArgs e)    {      isCompleted = false;      btnDownload.Enabled = false;      string url = txtUrl.Text.Trim();      string filePath = CreateFilePath(url);      #region 下載線程      Thread thread = new Thread(new ThreadStart(() =>      {        int tryTimes = 0;        while (!HttpDownloadFile(url, filePath))        {          Thread.Sleep(10000);          tryTimes++;          LogUtil.Log("請求服務器失敗，重新請求" + tryTimes.ToString() + "次");          this.Invoke(new InvokeDelegate(() =>          {            lblMsg.Text = "請求服務器失敗，重新請求" + tryTimes.ToString() + "次";          }));          HttpDownloadFile(url, filePath);        }      }));      thread.IsBackground = true;      thread.Start();      #endregion      #region 保存文件線程      thread = new Thread(new ThreadStart(() =>      {        while (!isCompleted)        {          MemoryStream ms;          if (msQueue.TryDequeue(out ms))          {            using (FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Write))            {              fs.Seek(completedCount, SeekOrigin.Begin);              fs.Write(ms.ToArray(), 0, (int)ms.Length);              fs.Close();            }            completedCount += ms.Length;          }          if (total != 0 && total == completedCount)          {            Thread.Sleep(100);            isCompleted = true;          }          Thread.Sleep(1);        }      }));      thread.IsBackground = true;      thread.Start();      #endregion      #region 計算網速/進度線程      thread = new Thread(new ThreadStart(() =>      {        while (!isCompleted)        {          Thread.Sleep(1000);          if (lastTime != DateTime.MinValue)          {            double sec = DateTime.Now.Subtract(lastTime).TotalSeconds;            double speed = unitCount / sec / 1024;            try            {              #region 顯示速度              if (speed < 1024)              {                this.Invoke(new InvokeDelegate(() =>                {                  lblSpeed.Text = string.Format("{0}KB/S", speed.ToString("0.00"));                }));              }              else              {                this.Invoke(new InvokeDelegate(() =>                {                  lblSpeed.Text = string.Format("{0}MB/S", (speed / 1024).ToString("0.00"));                }));              }              #endregion              #region 顯示進度              this.Invoke(new InvokeDelegate(() =>              {                string strTotal = (total / 1024 / 1024).ToString("0.00") + "MB";                if (total < 1024 * 1024)                {                  strTotal = (total / 1024).ToString("0.00") + "KB";                }                string completed = (completedCount / 1024 / 1024).ToString("0.00") + "MB";                if (completedCount < 1024 * 1024)                {                  completed = (completedCount / 1024).ToString("0.00") + "KB";                }                lblMsg.Text = string.Format("進度：{0}/{1}", completed, strTotal);                lblByteMsg.Text = string.Format("已下載：{0}/r/n總大小：{1}", completedCount, total);                if (completedCount == total)                {                  MessageBox.Show("完成");                }              }));              #endregion            }            catch { }            lastTime = DateTime.Now;            unitCount = 0;          }        }      }));      thread.IsBackground = true;      thread.Start();      #endregion      #region 限制網速線程      thread = new Thread(new ThreadStart(() =>      {        while (!isCompleted)        {          Thread.Sleep(100);          if (lastTimeForLimit != DateTime.MinValue)          {            double sec = DateTime.Now.Subtract(lastTimeForLimit).TotalSeconds;            double speed = unitCountForLimit / sec / 1024;            try            {              #region 限速/解除限速              double limitSpeed = 0;              if (double.TryParse(txtSpeed.Text.Trim(), out limitSpeed))              {                if (speed > limitSpeed && sleepTime < 1000)                {                  sleepTime += 1;                }                if (speed < limitSpeed - 10 && sleepTime >= 2)                {                  sleepTime -= 1;                }              }              else              {                this.Invoke(new InvokeDelegate(() =>                {                  txtSpeed.Text = "100";                }));              }              #endregion            }            catch { }            lastTimeForLimit = DateTime.Now;            unitCountForLimit = 0;          }        }      }));      thread.IsBackground = true;      thread.Start();      #endregion    }    #endregion    #region HttpDownloadFile 下載文件    /// <summary>    /// Http下載文件    /// </summary>    public bool HttpDownloadFile(string url, string filePath)    {      try      {        if (!File.Exists(filePath))        {          using (FileStream fs = new FileStream(filePath, FileMode.Create))          {            fs.Close();          }        }        else        {          FileInfo fileInfo = new FileInfo(filePath);          range = fileInfo.Length;        }        // 設置參數        HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;        request.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)";        request.Proxy = null;        //發送請求并獲取相應回應數據        HttpWebResponse response = request.GetResponse() as HttpWebResponse;        if (response.ContentLength == range)        {          this.Invoke(new InvokeDelegate(() =>          {            lblMsg.Text = "文件已下載";          }));          return true;        }        // 設置參數        request = WebRequest.Create(url) as HttpWebRequest;        request.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)";        request.Proxy = null;        request.AddRange(range);        //發送請求并獲取相應回應數據        response = request.GetResponse() as HttpWebResponse;        //直到request.GetResponse()程序才開始向目標網頁發送Post請求        Stream responseStream = response.GetResponseStream();        total = range + response.ContentLength;        completedCount = range;        MemoryStream ms = new MemoryStream();        byte[] bArr = new byte[1024];        lastTime = DateTime.Now;        lastTimeForLimit = DateTime.Now;        int size = responseStream.Read(bArr, 0, (int)bArr.Length);        unitCount += size;        unitCountForLimit += size;        ms.Write(bArr, 0, size);        while (!isCompleted)        {          size = responseStream.Read(bArr, 0, (int)bArr.Length);          unitCount += size;          unitCountForLimit += size;          ms.Write(bArr, 0, size);          if (ms.Length > 102400)          {            msQueue.Enqueue(ms);            ms = new MemoryStream();          }          if (completedCount + ms.Length == total)          {            msQueue.Enqueue(ms);            ms = new MemoryStream();          }          Thread.Sleep(sleepTime);        }        responseStream.Close();        return true;      }      catch (Exception ex)      {        LogUtil.LogError(ex.Message + "/r/n" + ex.StackTrace);        return false;      }    }    #endregion    #region 根據URL生成文件保存路徑    private string CreateFilePath(string url)    {      string path = Application.StartupPath + "http://download";      if (!Directory.Exists(path))      {        Directory.CreateDirectory(path);      }      string fileName = Path.GetFileName(url);      if (fileName.IndexOf("?") > 0)      {        return path + "http://" + fileName.Substring(0, fileName.IndexOf("?"));      }      else      {        return path + "http://" + fileName;      }    }    #endregion  } //end Form1類}