# 下載依賴yum install -y ncurses-devel openssl openssl-devel zlib-devel gcc make glibc-devel libffi-devel glibc-static glibc-utils sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-deve# 下載python版本wget https://www.python.org/ftp/python/3.5.1/Python-3.5.1.tgz# 或者使用國內源wget http://mirrors.sohu.com/python/3.5.1/Python-3.5.1.tgzmv Python-3.5.1.tgz /usr/local/src;cd /usr/local/src# 解壓tar -zxf Python-3.5.1.tgz;cd Python-3.5.1# 編譯安裝./configure --prefix=/usr/local/python3.5 --enable-sharedmake && make install# 建立軟鏈接ln -s /usr/local/python3.5/bin/python3 /usr/bin/python3echo "/usr/local/python3.5/lib" > /etc/ld.so.conf.d/python3.5.confldconfig# 驗證python3python3# Python 3.5.1 (default, Oct 9 2016, 11:44:24)# [GCC 4.8.5 20150623 (Red Hat 4.8.5-4)] on linux# Type "help", "copyright", "credits" or "license" for more information.# >>># pip/usr/local/python3.5/bin/pip3 install --upgrade pipln -s /usr/local/python3.5/bin/pip /usr/bin/pip# 本人在安裝時出現問題 將pip重裝wget https://bootstrap.pypa.io/get-pip.py --no-check-certificatepython get-pip.py

# 集成環境anaconda(推薦)wget https://repo.continuum.io/archive/Anaconda3-4.2.0-Linux-x86_64.sh# 直接安裝即可./Anaconda3-4.2.0-Linux-x86_64.sh# 若出錯，可能是解壓失敗yum install bzip2

# 安裝yum -y install mariadb mariadb-server# 啟動systemctl start mariadb# 設置為開機啟動systemctl enable mariadb# 配置密碼 默認為空mysql_secure_installation# 登錄mysql -u root -p# 創建一個用戶 自己設定賬戶密碼CREATE USER 'user_name'@'localhost' IDENTIFIED BY 'user_pass';GRANT ALL PRIVILEGES ON *.* TO 'user_name'@'localhost' WITH GRANT OPTION;CREATE USER 'user_name'@'%' IDENTIFIED BY 'user_pass';GRANT ALL PRIVILEGES ON *.* TO 'user_name'@'%' WITH GRANT OPTION;

# 搭建虛擬環境sbird python版本3.*conda create -n sbird python=3*# 進入環境source activate sbird# 安裝pyspiderpip install pyspider# 報錯 # it does not exist. The exported locale is "en_US.UTF-8" but it is not supported# 執行 可寫入.bashrcexport LC_ALL=en_US.utf-8export LANG=en_US.utf-8#ImportError: pycurl: libcurl link-time version (7.29.0) is older than compile-time version (7.49.0)conda install pycurl# 退出source deactivate sbird# 若在虛擬機內 出現無法訪問localhost:5000 可關閉防火墻systemctl stop firewalld.service#########直接運行源碼==============mkdir git;cd git# 下載git clone https://github.com/binux/pyspider.git# 安裝/root/anaconda3/envs/sbird/bin/python /root/git/pyspider/run.py

# 搭建虛擬環境pip install virtualenvmkdir python;cd python# 創建虛擬環境pyenv3virtualenv -p /usr/bin/python3 pyenv3# 進入虛擬環境 激活環境cd pyenv3/source ./bin/activatepip install pyspider# 若pycurl報錯 yum install libcurl-devel# 繼續pip install pyspider# 關閉deactivate

# 安裝yum install supervisor -y# 若無法檢索 則添加阿里的epel源vim /etc/yum.repos.d/epel.repo# 添加以下內容[epel]name=Extra Packages for Enterprise Linux 7 - $basearchbaseurl=http://mirrors.aliyun.com/epel/7/$basearchhttp://mirrors.aliyuncs.com/epel/7/$basearchfailovermethod=priorityenabled=1gpgcheck=0gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7[epel-debuginfo]name=Extra Packages for Enterprise Linux 7 - $basearch - Debugbaseurl=http://mirrors.aliyun.com/epel/7/$basearch/debughttp://mirrors.aliyuncs.com/epel/7/$basearch/debugfailovermethod=priorityenabled=0gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7gpgcheck=0[epel-source]name=Extra Packages for Enterprise Linux 7 - $basearch - Sourcebaseurl=http://mirrors.aliyun.com/epel/7/SRPMShttp://mirrors.aliyuncs.com/epel/7/SRPMSfailovermethod=priorityenabled=0gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7gpgcheck=0# 安裝yum install supervisor -y# 測試是否安裝成功echo_supervisord_conf

supervisord   #supervisor的服務器端部分 啟動supervisorctl  #啟動supervisor的命令行窗口# 假設創建進程pyspider01vim /etc/supervisord.d/pyspider01.ini# 寫入以下內容[program:pyspider01]command   = /root/anaconda3/envs/sbird/bin/python /root/git/pyspider/run.pydirectory  = /root/git/pyspideruser     = rootprocess_name = %(program_name)sautostart  = trueautorestart = truestartsecs  = 3redirect_stderr     = truestdout_logfile_maxbytes = 500MBstdout_logfile_backups = 10stdout_logfile     = /pyspider/supervisor/pyspider01.log# 重載supervisorctl reload# 啟動supervisorctl start pyspider01# 也可這樣啟動supervisord -c /etc/supervisord.conf# 查看狀態supervisorctl status# output pyspider01            RUNNING  pid 4026, uptime 0:02:40# 關閉supervisorctl shutdown

# 消息隊列采用redismkdir download;cd downloadwget http://download.redis.io/releases/redis-3.2.4.tar.gztar xzf redis-3.2.4.tar.gzcd redis-3.2.4make# 或者直接yum安裝yum -y install redis# 啟動systemctl start redis.service# 重啟systemctl restart redis.service# 停止systemctl stop redis.service# 查看狀態systemctl status redis.service# 更改文件/etc/redis.confvim /etc/redis.conf# 更改內容daemonize no 改為 daemonize yesbind 127.0.0.1 改為 bind 10.211.55.22(當前服務器ip)# 重啟redissystemctl restart redis.service

# Supervisor添加到自啟動服務systemctl enable supervisord.service# redis添加到自啟動服務systemctl enable redis.service# 關閉防火墻自啟動systemctl disable firewalld.service

centos01 10.211.55.22 redis,mariaDB, schedulercentos02 10.211.55.23 fetcher, processor, result_worker,phantomjscentos03 10.211.55.24 fetcher, processor,,result_worker,webui

{ "taskdb": "mysql+taskdb://user_name:user_pass@10.211.55.22:3306/taskdb", "projectdb": "mysql+projectdb://user_name:user_pass@10.211.55.22:3306/projectdb", "resultdb": "mysql+resultdb://user_name:user_pass@10.211.55.22:3306/resultdb", "message_queue": "redis://10.211.55.22:6379/db", "logging-config": "/pyspider/logging.conf", "phantomjs-proxy":"10.211.55.23:25555", "webui": {  "username": "",  "password": "",  "need-auth": false,  "host":"10.211.55.24",  "port":"5000",  "scheduler-rpc":"http:// 10.211.55.22:5002",  "fetcher-rpc":"http://10.211.55.23:5001" }, "fetcher": {  "xmlrpc":true,  "xmlrpc-host": "0.0.0.0",  "xmlrpc-port": "5001" }, "scheduler": {  "xmlrpc":true,  "xmlrpc-host": "0.0.0.0",  "xmlrpc-port": "5002" }}

/root/anaconda3/envs/sbird/bin/python /root/git/pyspider/run.py -c /pyspider/config.json scheduler# 報錯ImportError: No module named 'mysql'# 下載 mysql-connector-pythoncd ~/git/git clone https://github.com/mysql/mysql-connector-python.git# 安裝source activate sbirdcd mysql-connector-pythonpython setup.py install# 安裝redispip install redissource deactivate# 運行/root/anaconda3/envs/sbird/bin/python /root/git/pyspider/run.py -c /pyspider/config.json scheduler# 輸出 ok[I 161010 15:57:25 scheduler:644] scheduler starting...[I 161010 15:57:25 scheduler:779] scheduler.xmlrpc listening on 0.0.0.0:5002[I 161010 15:57:25 scheduler:583] in 5m: new:0,success:0,retry:0,failed:0

[program:pyspider01]command   = /root/anaconda3/envs/sbird/bin/python /root/git/pyspider/run.py -c /pyspider/config.json schedulerdirectory  = /root/git/pyspideruser     = rootprocess_name = %(program_name)sautostart  = trueautorestart = truestartsecs  = 3redirect_stderr     = truestdout_logfile_maxbytes = 500MBstdout_logfile_backups = 10stdout_logfile     = /pyspider/supervisor/pyspider01.log# 重載supervisorctl reload# 查看狀態supervisorctl status

/etc/supervisord.d/result_worker.ini[program:result_worker]command   = /root/anaconda3/envs/sbird/bin/python /root/git/pyspider/run.py -c /pyspider/config.json result_workerdirectory  = /root/git/pyspideruser     = rootprocess_name = %(program_name)sautostart  = trueautorestart = truestartsecs  = 3redirect_stderr     = truestdout_logfile_maxbytes = 500MBstdout_logfile_backups = 10stdout_logfile     = /pyspider/supervisor/result_worker.log/etc/supervisord.d/processor.ini[program:processor]command   = /root/anaconda3/envs/sbird/bin/python /root/git/pyspider/run.py -c /pyspider/config.json processordirectory  = /root/git/pyspideruser     = rootprocess_name = %(program_name)sautostart  = trueautorestart = truestartsecs  = 3redirect_stderr     = truestdout_logfile_maxbytes = 500MBstdout_logfile_backups = 10stdout_logfile     = /pyspider/supervisor/processor.log/etc/supervisord.d/phantomjs.ini[program:phantomjs]command   = /pyspider/phantomjs --config=/pyspider/pjsconfig.json /pyspider/phantomjs_fetcher.js 25555directory  = /root/git/pyspideruser     = rootprocess_name = %(program_name)sautostart  = trueautorestart = truestartsecs  = 3redirect_stderr     = truestdout_logfile_maxbytes = 500MBstdout_logfile_backups = 10stdout_logfile     = /pyspider/supervisor/phantomjs.log/etc/supervisord.d/fetcher.ini[program:fetcher]command   = /root/anaconda3/envs/sbird/bin/python /root/git/pyspider/run.py -c /pyspider/config.json fetcherdirectory  = /root/git/pyspideruser     = rootprocess_name = %(program_name)sautostart  = trueautorestart = truestartsecs  = 3redirect_stderr     = truestdout_logfile_maxbytes = 500MBstdout_logfile_backups = 10stdout_logfile     = /pyspider/supervisor/fetcher.log

{ /*--ignore-ssl-errors=true */ "ignoreSslErrors": true, /*--ssl-protocol=true */ "sslprotocol": "any", /* Same as: --output-encoding=utf8 */ "outputEncoding": "utf8", /* persistent Cookies. */ /*cookiesfile="e:/phontjscookies.txt",*/ cookiesfile="pyspider/phontjscookies.txt", /* load image */ autoLoadImages = false}

# 重載supervisorctl reload# 查看狀態supervisorctl status# outputfetcher             RUNNING  pid 3446, uptime 0:00:07phantomjs            RUNNING  pid 3448, uptime 0:00:07processor            RUNNING  pid 3447, uptime 0:00:07result_worker          RUNNING  pid 3445, uptime 0:00:07

/etc/supervisord.d/webui.ini[program:webui]command   = /root/anaconda3/envs/sbird/bin/python /root/git/pyspider/run.py -c /pyspider/config.json webuidirectory  = /root/git/pyspideruser     = rootprocess_name = %(program_name)sautostart  = trueautorestart = truestartsecs  = 3redirect_stderr     = truestdout_logfile_maxbytes = 500MBstdout_logfile_backups = 10stdout_logfile     = /pyspider/supervisor/webui.log# 重載supervisorctl reload# 查看狀態supervisorctl status# outputfetcher             RUNNING  pid 2724, uptime 0:00:07processor            RUNNING  pid 2725, uptime 0:00:07result_worker          RUNNING  pid 2723, uptime 0:00:07webui              RUNNING  pid 2726, uptime 0:00:07