By 王明军 on 2018年2月25日 • ( Leave a comment )

Python简易爬虫 – 抓取糗事百科糗事

这是之前博客里的一个简易爬虫代码，忘记导出博客了，所以从搜索引擎快照里扒拉了出来，毕竟这是自己的第一次python爬虫尝试。

<span style="color:#e7e77f;"> 1 </span>
<span style="color:#e7e77f;"> 2 </span><span style="color:#5ad5d5;font-weight:bold;">import</span> urllib<span style="color:#fbfbfb;">.</span>request
<span style="color:#e7e77f;"> 3 </span><span style="color:#5ad5d5;font-weight:bold;">import</span> urllib<span style="color:#fbfbfb;">.</span>parse
<span style="color:#e7e77f;"> 4 </span><span style="color:#5ad5d5;font-weight:bold;">import</span> time
<span style="color:#e7e77f;"> 5 </span><span style="color:#5ad5d5;font-weight:bold;">import</span> os
<span style="color:#e7e77f;"> 6 </span><span style="color:#5ad5d5;font-weight:bold;">import</span> threading
<span style="color:#e7e77f;"> 7 </span><span style="color:#5ad5d5;font-weight:bold;">import</span> queue
<span style="color:#e7e77f;"> 8 </span><span style="color:#5ad5d5;font-weight:bold;">import</span> bs4
<span style="color:#e7e77f;"> 9 </span><span style="color:#5ad5d5;font-weight:bold;">from</span> bs4 <span style="color:#5ad5d5;font-weight:bold;">import</span> BeautifulSoup
<span style="color:#e7e77f;">10 </span><span style="color:#5ad5d5;font-weight:bold;">import</span> shutil
<span style="color:#e7e77f;">11 </span><span style="color:#5ad5d5;font-weight:bold;">import</span> errno
<span style="color:#e7e77f;">12 </span><span style="color:#5ad5d5;font-weight:bold;">import</span> sys
<span style="color:#e7e77f;">13 </span>
<span style="color:#e7e77f;">14 </span>headers <span style="color:#fbfbfb;">= {</span><span style="color:#3fffa7;">'User-Agent'</span><span style="color:#fbfbfb;">:</span><span style="color:#3fffa7;">'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11'</span><span style="color:#fbfbfb;">,</span>
<span style="color:#e7e77f;">15 </span>                <span style="color:#3fffa7;">'Accept'</span><span style="color:#fbfbfb;">:</span><span style="color:#3fffa7;">'text/html;q=0.9,*/*;q=0.8'</span><span style="color:#fbfbfb;">,</span>
<span style="color:#e7e77f;">16 </span>                <span style="color:#3fffa7;">'Accept-Charset'</span><span style="color:#fbfbfb;">:</span><span style="color:#3fffa7;">'ISO-8859-1,utf-8;q=0.7,*;q=0.3'</span><span style="color:#fbfbfb;">,</span>
<span style="color:#e7e77f;">17 </span>                <span style="color:#3fffa7;">'Accept-Encoding'</span><span style="color:#fbfbfb;">:</span><span style="color:#3fffa7;">'gzip'</span><span style="color:#fbfbfb;">,</span>
<span style="color:#e7e77f;">18 </span>                <span style="color:#3fffa7;">'Connection'</span><span style="color:#fbfbfb;">:</span><span style="color:#3fffa7;">'close'</span><span style="color:#fbfbfb;">,</span>
<span style="color:#e7e77f;">19 </span>                <span style="color:#3fffa7;">'Referer'</span><span style="color:#fbfbfb;">:</span><span style="color:#5ad5d5;font-weight:bold;">None</span> <span style="color:#e7e77f;">#注意如果依然不能抓取，这里可以设置抓取网站的host</span>
<span style="color:#e7e77f;">20 </span>                <span style="color:#fbfbfb;">}</span>
<span style="color:#e7e77f;">21 </span>
<span style="color:#e7e77f;">22 </span><span style="color:#e7e77f;">#############################################################</span>
<span style="color:#e7e77f;">23 </span><span style="color:#e7e77f;"># working thread   </span>
<span style="color:#e7e77f;">24 </span><span style="color:#5ad5d5;font-weight:bold;">class</span> <span style="color:#ffffff;font-weight:bold;">Worker</span><span style="color:#fbfbfb;">(</span>threading<span style="color:#fbfbfb;">.</span>Thread<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">25 </span>       worker_count <span style="color:#fbfbfb;">=</span> <span style="color:#3fffa7;">0</span>
<span style="color:#e7e77f;">26 </span>
<span style="color:#e7e77f;">27 </span>       <span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">__init__</span><span style="color:#fbfbfb;">(</span> self<span style="color:#fbfbfb;">,</span> workQueue<span style="color:#fbfbfb;">,</span> resultQueue<span style="color:#fbfbfb;">,</span> timeout <span style="color:#fbfbfb;">=</span> <span style="color:#3fffa7;">0</span><span style="color:#fbfbfb;">, **</span>kwds<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">28 </span>           threading<span style="color:#fbfbfb;">.</span>Thread<span style="color:#3fffa7;">.__</span>init<span style="color:#3fffa7;">__</span><span style="color:#fbfbfb;">(</span> self<span style="color:#fbfbfb;">, **</span>kwds <span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">29 </span>           self<span style="color:#fbfbfb;">.</span><span style="color:#d174a8;font-weight:bold;">id</span> <span style="color:#fbfbfb;">=</span> Worker<span style="color:#fbfbfb;">.</span>worker_count
<span style="color:#e7e77f;">30 </span>           Worker<span style="color:#fbfbfb;">.</span>worker_count <span style="color:#fbfbfb;">+=</span> <span style="color:#3fffa7;">1</span>
<span style="color:#e7e77f;">31 </span>           self<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">setDaemon</span><span style="color:#fbfbfb;">(</span> <span style="color:#5ad5d5;font-weight:bold;">True</span> <span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">32 </span>           self<span style="color:#fbfbfb;">.</span>workQueue <span style="color:#fbfbfb;">=</span> workQueue
<span style="color:#e7e77f;">33 </span>           self<span style="color:#fbfbfb;">.</span>resultQueue <span style="color:#fbfbfb;">=</span> resultQueue
<span style="color:#e7e77f;">34 </span>           self<span style="color:#fbfbfb;">.</span>timeout <span style="color:#fbfbfb;">=</span> timeout
<span style="color:#e7e77f;">35 </span>           self<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">start</span><span style="color:#fbfbfb;">()</span>
<span style="color:#e7e77f;">36 </span>
<span style="color:#e7e77f;">37 </span>       <span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">run</span><span style="color:#fbfbfb;">(</span> self <span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">38 </span>           <span style="color:#3fffa7;">''' the get-some-work, do-some-work main loop of worker threads '''</span>
<span style="color:#e7e77f;">39 </span>           <span style="color:#5ad5d5;font-weight:bold;">while True</span><span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">40 </span>              <span style="color:#5ad5d5;font-weight:bold;">try</span><span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">41 </span>                  <span style="color:#d174a8;font-weight:bold;">callable</span><span style="color:#fbfbfb;">,</span> args<span style="color:#fbfbfb;">,</span> kwds <span style="color:#fbfbfb;">=</span> self<span style="color:#fbfbfb;">.</span>workQueue<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">get</span><span style="color:#fbfbfb;">()</span>
<span style="color:#e7e77f;">42 </span>                  res <span style="color:#fbfbfb;">=</span> <span style="color:#d174a8;font-weight:bold;">callable</span><span style="color:#fbfbfb;">(*</span>args<span style="color:#fbfbfb;">, **</span>kwds<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">43 </span>                  <span style="color:#e7e77f;">#print("worker[%2d]: %s" % (self.id, str(res) ))</span>
<span style="color:#e7e77f;">44 </span>                  <span style="color:#e7e77f;">#self.resultQueue.put( res )</span>
<span style="color:#e7e77f;">45 </span>              <span style="color:#5ad5d5;font-weight:bold;">except</span> queue<span style="color:#fbfbfb;">.</span>Empty<span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">46 </span>                  <span style="color:#5ad5d5;font-weight:bold;">pass</span>
<span style="color:#e7e77f;">47 </span>                  <span style="color:#e7e77f;">#break</span>
<span style="color:#e7e77f;">48 </span>              <span style="color:#5ad5d5;font-weight:bold;">except</span><span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">49 </span>                  <span style="color:#5ad5d5;font-weight:bold;">print</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'worker[%2d]'</span> <span style="color:#fbfbfb;">%</span> self<span style="color:#fbfbfb;">.</span><span style="color:#d174a8;font-weight:bold;">id</span><span style="color:#fbfbfb;">,</span> sys<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">exc_info</span><span style="color:#fbfbfb;">()[:</span><span style="color:#3fffa7;">2</span><span style="color:#fbfbfb;">])</span>   
<span style="color:#e7e77f;">50 </span>                     
<span style="color:#e7e77f;">51 </span><span style="color:#5ad5d5;font-weight:bold;">class</span> WorkerManager<span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">52 </span>       <span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">__init__</span><span style="color:#fbfbfb;">(</span>self<span style="color:#fbfbfb;">,</span> num_of_workers<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">10</span><span style="color:#fbfbfb;">,</span> timeout <span style="color:#fbfbfb;">=</span> <span style="color:#3fffa7;">1</span><span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">53 </span>           self<span style="color:#fbfbfb;">.</span>workQueue <span style="color:#fbfbfb;">=</span> queue<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">Queue</span><span style="color:#fbfbfb;">()</span>
<span style="color:#e7e77f;">54 </span>           self<span style="color:#fbfbfb;">.</span>resultQueue <span style="color:#fbfbfb;">=</span> queue<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">Queue</span><span style="color:#fbfbfb;">()</span>
<span style="color:#e7e77f;">55 </span>           self<span style="color:#fbfbfb;">.</span>workers <span style="color:#fbfbfb;">= []</span>
<span style="color:#e7e77f;">56 </span>           self<span style="color:#fbfbfb;">.</span>timeout <span style="color:#fbfbfb;">=</span> timeout
<span style="color:#e7e77f;">57 </span>           self<span style="color:#3fffa7;">._</span>recruitThreads<span style="color:#fbfbfb;">(</span>num_of_workers<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">58 </span>
<span style="color:#e7e77f;">59 </span>       <span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">_recruitThreads</span><span style="color:#fbfbfb;">(</span>self<span style="color:#fbfbfb;">,</span> num_of_workers<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">60 </span>          <span style="color:#5ad5d5;font-weight:bold;">for</span> i <span style="color:#5ad5d5;font-weight:bold;">in</span> <span style="color:#d174a8;font-weight:bold;">range</span><span style="color:#fbfbfb;">(</span>num_of_workers<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">61 </span>              worker <span style="color:#fbfbfb;">=</span> <span style="color:#ffffff;font-weight:bold;">Worker</span><span style="color:#fbfbfb;">(</span>self<span style="color:#fbfbfb;">.</span>workQueue<span style="color:#fbfbfb;">,</span> self<span style="color:#fbfbfb;">.</span>resultQueue<span style="color:#fbfbfb;">,</span> self<span style="color:#fbfbfb;">.</span>timeout<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">62 </span>              self<span style="color:#fbfbfb;">.</span>workers<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">append</span><span style="color:#fbfbfb;">(</span>worker<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">63 </span>
<span style="color:#e7e77f;">64 </span>       <span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">wait_for_complete</span><span style="color:#fbfbfb;">(</span>self<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">65 </span>           <span style="color:#e7e77f;"># ...then, wait for each of them to terminate:</span>
<span style="color:#e7e77f;">66 </span>           <span style="color:#5ad5d5;font-weight:bold;">while</span> <span style="color:#d174a8;font-weight:bold;">len</span><span style="color:#fbfbfb;">(</span>self<span style="color:#fbfbfb;">.</span>workers<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">67 </span>               worker <span style="color:#fbfbfb;">=</span> self<span style="color:#fbfbfb;">.</span>workers<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">pop</span><span style="color:#fbfbfb;">()</span>
<span style="color:#e7e77f;">68 </span>               worker<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">join</span><span style="color:#fbfbfb;">()</span>
<span style="color:#e7e77f;">69 </span>               <span style="color:#5ad5d5;font-weight:bold;">if</span> worker<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">isAlive</span><span style="color:#fbfbfb;">()</span> <span style="color:#5ad5d5;font-weight:bold;">and not</span> self<span style="color:#fbfbfb;">.</span>workQueue<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">empty</span><span style="color:#fbfbfb;">():</span>
<span style="color:#e7e77f;">70 </span>                   self<span style="color:#fbfbfb;">.</span>workers<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">append</span><span style="color:#fbfbfb;">(</span> worker <span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">71 </span>           <span style="color:#5ad5d5;font-weight:bold;">print</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">"All jobs are are completed."</span><span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">72 </span>
<span style="color:#e7e77f;">73 </span>       <span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">add_job</span><span style="color:#fbfbfb;">(</span> self<span style="color:#fbfbfb;">,</span> <span style="color:#d174a8;font-weight:bold;">callable</span><span style="color:#fbfbfb;">, *</span>args<span style="color:#fbfbfb;">, **</span>kwds <span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">74 </span>           self<span style="color:#fbfbfb;">.</span>workQueue<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">put</span><span style="color:#fbfbfb;">( (</span><span style="color:#d174a8;font-weight:bold;">callable</span><span style="color:#fbfbfb;">,</span> args<span style="color:#fbfbfb;">,</span> kwds<span style="color:#fbfbfb;">) )</span>
<span style="color:#e7e77f;">75 </span>
<span style="color:#e7e77f;">76 </span>       <span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">get_result</span><span style="color:#fbfbfb;">(</span> self<span style="color:#fbfbfb;">, *</span>args<span style="color:#fbfbfb;">, **</span>kwds <span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">77 </span>           <span style="color:#5ad5d5;font-weight:bold;">return</span> self<span style="color:#fbfbfb;">.</span>resultQueue<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">get</span><span style="color:#fbfbfb;">( *</span>args<span style="color:#fbfbfb;">, **</span>kwds <span style="color:#fbfbfb;">)</span>  
<span style="color:#e7e77f;">78 </span>
<span style="color:#e7e77f;">79 </span>
<span style="color:#e7e77f;">80 </span>
<span style="color:#e7e77f;">81 </span><span style="color:#e7e77f;">#############################################################</span>
<span style="color:#e7e77f;">82 </span>
<span style="color:#e7e77f;">83 </span>
<span style="color:#e7e77f;">84 </span><span style="color:#5ad5d5;font-weight:bold;">class</span> Fetcher<span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">85 </span>       <span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">__init__</span><span style="color:#fbfbfb;">(</span>self<span style="color:#fbfbfb;">,</span>manager<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">86 </span>           self<span style="color:#fbfbfb;">.</span>opener <span style="color:#fbfbfb;">=</span> urllib<span style="color:#fbfbfb;">.</span>request<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">build_opener</span><span style="color:#fbfbfb;">(</span>urllib<span style="color:#fbfbfb;">.</span>request<span style="color:#fbfbfb;">.</span>HTTPHandler<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">87 </span>           self<span style="color:#fbfbfb;">.</span>opener<span style="color:#fbfbfb;">.</span>addheaders <span style="color:#fbfbfb;">= [(</span><span style="color:#3fffa7;">'User-agent'</span><span style="color:#fbfbfb;">,</span> <span style="color:#3fffa7;">'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11'</span><span style="color:#fbfbfb;">)]</span>
<span style="color:#e7e77f;">88 </span>           self<span style="color:#fbfbfb;">.</span>lock <span style="color:#fbfbfb;">=</span> threading<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">Lock</span><span style="color:#fbfbfb;">()</span> <span style="color:#e7e77f;">#线程锁</span>
<span style="color:#e7e77f;">89 </span>           self<span style="color:#fbfbfb;">.</span>q_req <span style="color:#fbfbfb;">=</span> queue<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">Queue</span><span style="color:#fbfbfb;">()</span> <span style="color:#e7e77f;">#任务队列</span>
<span style="color:#e7e77f;">90 </span>           self<span style="color:#fbfbfb;">.</span>q_ans <span style="color:#fbfbfb;">=</span> queue<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">Queue</span><span style="color:#fbfbfb;">()</span> <span style="color:#e7e77f;">#完成队列</span>
<span style="color:#e7e77f;">91 </span>           self<span style="color:#3fffa7;">.__</span>q<span style="color:#3fffa7;">_</span>retry <span style="color:#fbfbfb;">=</span> queue<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">Queue</span><span style="color:#fbfbfb;">()</span> <span style="color:#e7e77f;">#重试队列</span>
<span style="color:#e7e77f;">92 </span>           <span style="color:#e7e77f;">#self.threads_num = threads_num + 1 # 1代表了重试线程</span>
<span style="color:#e7e77f;">93 </span>           <span style="color:#e7e77f;">#self.__threads = []</span>
<span style="color:#e7e77f;">94 </span>
<span style="color:#e7e77f;">95 </span>           <span style="color:#e7e77f;">#重试线程</span>
<span style="color:#e7e77f;">96 </span>           manager<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">add_job</span><span style="color:#fbfbfb;">(</span>self<span style="color:#fbfbfb;">.</span>threadretry<span style="color:#fbfbfb;">,</span>self<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">97 </span>           <span style="color:#e7e77f;">#self.__retry_thread = threading.Thread(target=self.threadretry)</span>
<span style="color:#e7e77f;">98 </span>           <span style="color:#e7e77f;">#self.__threads.append(self.__retry_thread)</span>
<span style="color:#e7e77f;">99 </span>           <span style="color:#e7e77f;">#self.__retry_thread.setDaemon(True)</span>
<span style="color:#e7e77f;">100 </span>           <span style="color:#e7e77f;">#self.__retry_thread.start()</span>
<span style="color:#e7e77f;">101 </span>
<span style="color:#e7e77f;">102 </span>           <span style="color:#e7e77f;">#开启@threads_num个工作线程</span>
<span style="color:#e7e77f;">103 </span>           manager<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">add_job</span><span style="color:#fbfbfb;">(</span>self<span style="color:#fbfbfb;">.</span>threadget<span style="color:#fbfbfb;">,</span>self<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">104 </span>           <span style="color:#e7e77f;">#for i in range(threads_num):</span>
<span style="color:#e7e77f;">105 </span>           <span style="color:#e7e77f;">#   t = threading.Thread(target=self.threadget)</span>
<span style="color:#e7e77f;">106 </span>           <span style="color:#e7e77f;">#   self.__threads.append(t)</span>
<span style="color:#e7e77f;">107 </span>           <span style="color:#e7e77f;">#   t.setDaemon(True)</span>
<span style="color:#e7e77f;">108 </span>           <span style="color:#e7e77f;">#   t.start()</span>
<span style="color:#e7e77f;">109 </span>               
<span style="color:#e7e77f;">110 </span>           self<span style="color:#fbfbfb;">.</span>running <span style="color:#fbfbfb;">=</span> <span style="color:#3fffa7;">0</span>
<span style="color:#e7e77f;">111 </span>
<span style="color:#e7e77f;">112 </span>       <span style="color:#e7e77f;">#def __del__(self): #解构时需等待两个队列完成</span>
<span style="color:#e7e77f;">113 </span>           <span style="color:#e7e77f;">#print("will delete")</span>
<span style="color:#e7e77f;">114 </span>           <span style="color:#e7e77f;">#self.q_req.join()</span>
<span style="color:#e7e77f;">115 </span>           <span style="color:#e7e77f;">#self.q_ans.join()</span>
<span style="color:#e7e77f;">116 </span>           <span style="color:#e7e77f;">#self.__q_retry.join()</span>
<span style="color:#e7e77f;">117 </span>           <span style="color:#e7e77f;">#print("deleted")</span>
<span style="color:#e7e77f;">118 </span>           <span style="color:#e7e77f;">#for i in range(self.threads_num+1):</span>
<span style="color:#e7e77f;">119 </span>           <span style="color:#e7e77f;">#    self.__threads[i].exit()</span>
<span style="color:#e7e77f;">120 </span>           <span style="color:#e7e77f;">#print("还有"+str(threading.active_count())+"个活跃线程")</span>
<span style="color:#e7e77f;">121 </span>
<span style="color:#e7e77f;">122 </span>       <span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">taskleft</span><span style="color:#fbfbfb;">(</span>self<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">123 </span>           <span style="color:#5ad5d5;font-weight:bold;">return</span> self<span style="color:#fbfbfb;">.</span>q_req<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">qsize</span><span style="color:#fbfbfb;">()+</span>self<span style="color:#fbfbfb;">.</span>q_ans<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">qsize</span><span style="color:#fbfbfb;">()+</span>self<span style="color:#fbfbfb;">.</span>running
<span style="color:#e7e77f;">124 </span>
<span style="color:#e7e77f;">125 </span>       <span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">push</span><span style="color:#fbfbfb;">(</span>self<span style="color:#fbfbfb;">,</span>req<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">126 </span>           self<span style="color:#fbfbfb;">.</span>q_req<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">put</span><span style="color:#fbfbfb;">(</span>req<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">127 </span>
<span style="color:#e7e77f;">128 </span>       <span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">pop</span><span style="color:#fbfbfb;">(</span>self<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">129 </span>           <span style="color:#5ad5d5;font-weight:bold;">return</span> self<span style="color:#fbfbfb;">.</span>q_ans<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">get</span><span style="color:#fbfbfb;">()</span>
<span style="color:#e7e77f;">130 </span>
<span style="color:#e7e77f;">131 </span>       <span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">threadretry</span><span style="color:#fbfbfb;">(</span>self<span style="color:#fbfbfb;">,</span>_self<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">132 </span>           <span style="color:#5ad5d5;font-weight:bold;">while True</span><span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">133 </span>               <span style="color:#5ad5d5;font-weight:bold;">try</span><span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">134 </span>                   req <span style="color:#fbfbfb;">=</span> _self<span style="color:#3fffa7;">.__</span>q<span style="color:#3fffa7;">_</span>retry<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">get</span><span style="color:#fbfbfb;">(</span>timeout<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">1</span><span style="color:#fbfbfb;">)</span> <span style="color:#e7e77f;">#get(self,block=True,timeout=None)</span>
<span style="color:#e7e77f;">135 </span>               <span style="color:#5ad5d5;font-weight:bold;">except</span> queue<span style="color:#fbfbfb;">.</span>Empty<span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">136 </span>                   <span style="color:#5ad5d5;font-weight:bold;">break</span>
<span style="color:#e7e77f;">137 </span>               
<span style="color:#e7e77f;">138 </span>               <span style="color:#e7e77f;">#with _self.lock:</span>
<span style="color:#e7e77f;">139 </span>               _self<span style="color:#fbfbfb;">.</span>running <span style="color:#fbfbfb;">+=</span> <span style="color:#3fffa7;">1</span>
<span style="color:#e7e77f;">140 </span>
<span style="color:#e7e77f;">141 </span>               <span style="color:#5ad5d5;font-weight:bold;">try</span><span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">142 </span>                   ans <span style="color:#fbfbfb;">=</span> _self<span style="color:#fbfbfb;">.</span>opener<span style="color:#fbfbfb;">.</span><span style="color:#d174a8;font-weight:bold;">open</span><span style="color:#fbfbfb;">(</span>req<span style="color:#fbfbfb;">).</span><span style="color:#ffffff;font-weight:bold;">read</span><span style="color:#fbfbfb;">()</span>
<span style="color:#e7e77f;">143 </span>
<span style="color:#e7e77f;">144 </span>               <span style="color:#5ad5d5;font-weight:bold;">except</span> urllib<span style="color:#fbfbfb;">.</span>error<span style="color:#fbfbfb;">.</span>URLError <span style="color:#5ad5d5;font-weight:bold;">as</span> e<span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">145 </span>                   ans <span style="color:#fbfbfb;">=</span> <span style="color:#3fffa7;">''</span>
<span style="color:#e7e77f;">146 </span>                   <span style="color:#5ad5d5;font-weight:bold;">if</span> <span style="color:#d174a8;font-weight:bold;">hasattr</span><span style="color:#fbfbfb;">(</span>e<span style="color:#fbfbfb;">,</span> <span style="color:#3fffa7;">'reason'</span><span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">147 </span>                       <span style="color:#5ad5d5;font-weight:bold;">print</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'We failed to reach a server.'</span><span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">148 </span>                       <span style="color:#5ad5d5;font-weight:bold;">print</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'Reason: '</span><span style="color:#fbfbfb;">,</span> e<span style="color:#fbfbfb;">.</span>reason<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">149 </span>                   <span style="color:#5ad5d5;font-weight:bold;">elif</span> <span style="color:#d174a8;font-weight:bold;">hasattr</span><span style="color:#fbfbfb;">(</span>e<span style="color:#fbfbfb;">,</span> <span style="color:#3fffa7;">'code'</span><span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">150 </span>                       <span style="color:#5ad5d5;font-weight:bold;">print</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'The server cannot fulfill the request.'</span><span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">151 </span>                       <span style="color:#5ad5d5;font-weight:bold;">print</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'Reason: '</span><span style="color:#fbfbfb;">,</span> e<span style="color:#fbfbfb;">.</span>code<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">152 </span>               <span style="color:#5ad5d5;font-weight:bold;">else</span><span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">153 </span>                   <span style="color:#5ad5d5;font-weight:bold;">if</span> ans<span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">154 </span>                       _self<span style="color:#fbfbfb;">.</span>q_ans<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">put</span><span style="color:#fbfbfb;">((</span>req<span style="color:#fbfbfb;">,</span>ans<span style="color:#fbfbfb;">))</span>
<span style="color:#e7e77f;">155 </span>               <span style="color:#5ad5d5;font-weight:bold;">finally</span><span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">156 </span>                   <span style="color:#e7e77f;">#with _self.lock:</span>
<span style="color:#e7e77f;">157 </span>                   _self<span style="color:#fbfbfb;">.</span>running <span style="color:#fbfbfb;">-=</span> <span style="color:#3fffa7;">1</span>
<span style="color:#e7e77f;">158 </span>
<span style="color:#e7e77f;">159 </span>                   _self<span style="color:#3fffa7;">.__</span>q<span style="color:#3fffa7;">_</span>retry<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">task_done</span><span style="color:#fbfbfb;">()</span>
<span style="color:#e7e77f;">160 </span>               
<span style="color:#e7e77f;">161 </span>
<span style="color:#e7e77f;">162 </span>
<span style="color:#e7e77f;">163 </span>       <span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">threadget</span><span style="color:#fbfbfb;">(</span>self<span style="color:#fbfbfb;">,</span>_self<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">164 </span>           <span style="color:#5ad5d5;font-weight:bold;">while True</span><span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">165 </span>               <span style="color:#5ad5d5;font-weight:bold;">try</span><span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">166 </span>                   req <span style="color:#fbfbfb;">=</span> _self<span style="color:#fbfbfb;">.</span>q_req<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">get</span><span style="color:#fbfbfb;">(</span>timeout<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">1</span><span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">167 </span>               <span style="color:#5ad5d5;font-weight:bold;">except</span> queue<span style="color:#fbfbfb;">.</span>Empty<span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">168 </span>                   <span style="color:#5ad5d5;font-weight:bold;">break</span>
<span style="color:#e7e77f;">169 </span>               <span style="color:#e7e77f;">#with _self.lock: #要保证该操作的原子性，进入critical area</span>
<span style="color:#e7e77f;">170 </span>               _self<span style="color:#fbfbfb;">.</span>running <span style="color:#fbfbfb;">+=</span> <span style="color:#3fffa7;">1</span>
<span style="color:#e7e77f;">171 </span>                   
<span style="color:#e7e77f;">172 </span>               <span style="color:#5ad5d5;font-weight:bold;">try</span><span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">173 </span>                   ans <span style="color:#fbfbfb;">=</span> _self<span style="color:#fbfbfb;">.</span>opener<span style="color:#fbfbfb;">.</span><span style="color:#d174a8;font-weight:bold;">open</span><span style="color:#fbfbfb;">(</span>req<span style="color:#fbfbfb;">).</span><span style="color:#ffffff;font-weight:bold;">read</span><span style="color:#fbfbfb;">()</span>
<span style="color:#e7e77f;">174 </span>               <span style="color:#5ad5d5;font-weight:bold;">except</span> urllib<span style="color:#fbfbfb;">.</span>error<span style="color:#fbfbfb;">.</span>URLError <span style="color:#5ad5d5;font-weight:bold;">as</span> e<span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">175 </span>                   ans <span style="color:#fbfbfb;">=</span> <span style="color:#3fffa7;">''</span>
<span style="color:#e7e77f;">176 </span>                   <span style="color:#5ad5d5;font-weight:bold;">if</span> <span style="color:#d174a8;font-weight:bold;">hasattr</span><span style="color:#fbfbfb;">(</span>e<span style="color:#fbfbfb;">,</span> <span style="color:#3fffa7;">'reason'</span><span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">177 </span>                       <span style="color:#5ad5d5;font-weight:bold;">print</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'We failed to reach a server.'</span><span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">178 </span>                       <span style="color:#5ad5d5;font-weight:bold;">print</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'Reason: '</span><span style="color:#fbfbfb;">,</span> e<span style="color:#fbfbfb;">.</span>reason<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">179 </span>                       _self<span style="color:#3fffa7;">.__</span>q<span style="color:#3fffa7;">_</span>retry<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">put</span><span style="color:#fbfbfb;">(</span>req<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">180 </span>                   <span style="color:#5ad5d5;font-weight:bold;">elif</span> <span style="color:#d174a8;font-weight:bold;">hasattr</span><span style="color:#fbfbfb;">(</span>e<span style="color:#fbfbfb;">,</span> <span style="color:#3fffa7;">'code'</span><span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">181 </span>                       <span style="color:#5ad5d5;font-weight:bold;">print</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'The server cannot fulfill the request.'</span><span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">182 </span>                       <span style="color:#5ad5d5;font-weight:bold;">print</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'Reason: '</span><span style="color:#fbfbfb;">,</span> e<span style="color:#fbfbfb;">.</span>code<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">183 </span>                       _self<span style="color:#3fffa7;">.__</span>q<span style="color:#3fffa7;">_</span>retry<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">put</span><span style="color:#fbfbfb;">(</span>req<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">184 </span>               <span style="color:#5ad5d5;font-weight:bold;">else</span><span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">185 </span>                   <span style="color:#5ad5d5;font-weight:bold;">if</span> ans<span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">186 </span>                       _self<span style="color:#fbfbfb;">.</span>q_ans<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">put</span><span style="color:#fbfbfb;">((</span>req<span style="color:#fbfbfb;">,</span>ans<span style="color:#fbfbfb;">))</span>
<span style="color:#e7e77f;">187 </span>                   <span style="color:#5ad5d5;font-weight:bold;">else</span><span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">188 </span>                       _self<span style="color:#3fffa7;">.__</span>q<span style="color:#3fffa7;">_</span>retry<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">put</span><span style="color:#fbfbfb;">(</span>req<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">189 </span>               <span style="color:#5ad5d5;font-weight:bold;">finally</span><span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">190 </span>                   <span style="color:#e7e77f;">#with _self.lock:</span>
<span style="color:#e7e77f;">191 </span>                   _self<span style="color:#fbfbfb;">.</span>running <span style="color:#fbfbfb;">-=</span> <span style="color:#3fffa7;">1</span>
<span style="color:#e7e77f;">192 </span>                   _self<span style="color:#fbfbfb;">.</span>q_req<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">task_done</span><span style="color:#fbfbfb;">()</span>
<span style="color:#e7e77f;">193 </span>               
<span style="color:#e7e77f;">194 </span>
<span style="color:#e7e77f;">195 </span><span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">create_dir</span><span style="color:#fbfbfb;">(</span>userid<span style="color:#fbfbfb;">,</span>domain<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">'qiushibaike'</span><span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">196 </span>       dir_name <span style="color:#fbfbfb;">=</span> domain <span style="color:#fbfbfb;">+</span> <span style="color:#3fffa7;">'/'</span> <span style="color:#fbfbfb;">+</span> userid
<span style="color:#e7e77f;">197 </span>       <span style="color:#5ad5d5;font-weight:bold;">try</span><span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">198 </span>           os<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">mkdir</span><span style="color:#fbfbfb;">(</span>dir_name<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">199 </span>       <span style="color:#5ad5d5;font-weight:bold;">except</span> <span style="color:#98d65a;font-weight:bold;">OSError</span> <span style="color:#5ad5d5;font-weight:bold;">as</span> e<span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">200 </span>           <span style="color:#5ad5d5;font-weight:bold;">if</span> e<span style="color:#fbfbfb;">.</span>errno <span style="color:#fbfbfb;">==</span> errno<span style="color:#fbfbfb;">.</span>EEXIST <span style="color:#5ad5d5;font-weight:bold;">and</span> os<span style="color:#fbfbfb;">.</span>path<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">isdir</span><span style="color:#fbfbfb;">(</span>dir_name<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">201 </span>               <span style="color:#5ad5d5;font-weight:bold;">pass</span>
<span style="color:#e7e77f;">202 </span>           <span style="color:#5ad5d5;font-weight:bold;">else</span><span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">203 </span>               <span style="color:#5ad5d5;font-weight:bold;">print</span><span style="color:#fbfbfb;">(</span><span style="color:#d174a8;font-weight:bold;">str</span><span style="color:#fbfbfb;">(</span>e<span style="color:#fbfbfb;">))</span>
<span style="color:#e7e77f;">204 </span>
<span style="color:#e7e77f;">205 </span><span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">userid_exist</span><span style="color:#fbfbfb;">(</span>userid<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">206 </span>       <span style="color:#5ad5d5;font-weight:bold;">return</span> os<span style="color:#fbfbfb;">.</span>path<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">isdir</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'qiushibaike'</span><span style="color:#fbfbfb;">+</span><span style="color:#3fffa7;">'/'</span><span style="color:#fbfbfb;">+</span>userid<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">207 </span>
<span style="color:#e7e77f;">208 </span><span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">get_file_name</span><span style="color:#fbfbfb;">(</span>userid<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">209 </span>       current_time <span style="color:#fbfbfb;">=</span> time<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">strftime</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">"%Y-%m-</span><span style="color:#bfbfef;">%d</span><span style="color:#3fffa7;">"</span><span style="color:#fbfbfb;">,</span>time<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">localtime</span><span style="color:#fbfbfb;">())</span>
<span style="color:#e7e77f;">210 </span>       <span style="color:#5ad5d5;font-weight:bold;">return</span> <span style="color:#3fffa7;">'qiushibaike'</span><span style="color:#fbfbfb;">+</span><span style="color:#3fffa7;">'/'</span><span style="color:#fbfbfb;">+</span>userid<span style="color:#fbfbfb;">+</span><span style="color:#3fffa7;">'/'</span><span style="color:#fbfbfb;">+</span>current_time<span style="color:#fbfbfb;">+</span><span style="color:#3fffa7;">'.txt'</span>
<span style="color:#e7e77f;">211 </span>
<span style="color:#e7e77f;">212 </span>
<span style="color:#e7e77f;">213 </span>
<span style="color:#e7e77f;">214 </span><span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">write_file</span><span style="color:#fbfbfb;">(</span><span style="color:#d174a8;font-weight:bold;">file</span><span style="color:#fbfbfb;">,</span>soup<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">215 </span>       count<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">0</span>
<span style="color:#e7e77f;">216 </span>       <span style="color:#5ad5d5;font-weight:bold;">for</span> ii <span style="color:#5ad5d5;font-weight:bold;">in</span> soup<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">find_all</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">"div"</span><span style="color:#fbfbfb;">,</span>class_<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">"content clearfix"</span><span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">217 </span>           <span style="color:#e7e77f;">#print(ii.a["href"])</span>
<span style="color:#e7e77f;">218 </span>           <span style="color:#e7e77f;">#print(ii.a.text)</span>
<span style="color:#e7e77f;">219 </span>           <span style="color:#5ad5d5;font-weight:bold;">if</span> ii<span style="color:#fbfbfb;">.</span>a<span style="color:#fbfbfb;">.</span>text<span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">220 </span>               count <span style="color:#fbfbfb;">+=</span> <span style="color:#3fffa7;">1</span>
<span style="color:#e7e77f;">221 </span>               <span style="color:#d174a8;font-weight:bold;">file</span><span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">write</span><span style="color:#fbfbfb;">(</span><span style="color:#ffffff;font-weight:bold;">bytes</span><span style="color:#fbfbfb;">(</span>ii<span style="color:#fbfbfb;">.</span>a<span style="color:#fbfbfb;">[</span><span style="color:#3fffa7;">"href"</span><span style="color:#fbfbfb;">],</span>encoding<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">"utf-8"</span><span style="color:#fbfbfb;">))</span>
<span style="color:#e7e77f;">222 </span>               <span style="color:#d174a8;font-weight:bold;">file</span><span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">write</span><span style="color:#fbfbfb;">(</span><span style="color:#ffffff;font-weight:bold;">bytes</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'</span><span style="color:#bfbfef;">rn</span><span style="color:#3fffa7;">'</span><span style="color:#fbfbfb;">,</span>encoding<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">"utf-8"</span><span style="color:#fbfbfb;">))</span>
<span style="color:#e7e77f;">223 </span>               <span style="color:#d174a8;font-weight:bold;">file</span><span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">write</span><span style="color:#fbfbfb;">(</span><span style="color:#ffffff;font-weight:bold;">bytes</span><span style="color:#fbfbfb;">(</span>ii<span style="color:#fbfbfb;">.</span>a<span style="color:#fbfbfb;">.</span>text<span style="color:#fbfbfb;">,</span>encoding<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">"utf-8"</span><span style="color:#fbfbfb;">))</span>
<span style="color:#e7e77f;">224 </span>               <span style="color:#d174a8;font-weight:bold;">file</span><span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">write</span><span style="color:#fbfbfb;">(</span><span style="color:#ffffff;font-weight:bold;">bytes</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">"</span><span style="color:#bfbfef;">rnrn</span><span style="color:#3fffa7;">"</span><span style="color:#fbfbfb;">,</span>encoding<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">"utf-8"</span><span style="color:#fbfbfb;">))</span>
<span style="color:#e7e77f;">225 </span>       <span style="color:#5ad5d5;font-weight:bold;">return</span> count
<span style="color:#e7e77f;">226 </span>       
<span style="color:#e7e77f;">227 </span><span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">get_max_page</span><span style="color:#fbfbfb;">(</span>soup<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">228 </span>       <span style="color:#e7e77f;">#ii=bs4.element.Tag()</span>
<span style="color:#e7e77f;">229 </span>       num<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">0</span>
<span style="color:#e7e77f;">230 </span>       <span style="color:#5ad5d5;font-weight:bold;">for</span> jj <span style="color:#5ad5d5;font-weight:bold;">in</span> soup<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">find_all</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'a'</span><span style="color:#fbfbfb;">,</span>rel<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">"next"</span><span style="color:#fbfbfb;">,</span>class_<span style="color:#fbfbfb;">=</span><span style="color:#5ad5d5;font-weight:bold;">None</span><span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">231 </span>           num<span style="color:#fbfbfb;">=</span><span style="color:#d174a8;font-weight:bold;">int</span><span style="color:#fbfbfb;">(</span>jj<span style="color:#fbfbfb;">.</span>text<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">232 </span>       <span style="color:#5ad5d5;font-weight:bold;">return</span> num
<span style="color:#e7e77f;">233 </span>
<span style="color:#e7e77f;">234 </span>               
<span style="color:#e7e77f;">235 </span><span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">store_this_user</span><span style="color:#fbfbfb;">(</span>userid<span style="color:#fbfbfb;">,</span>manager<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">236 </span>       <span style="color:#5ad5d5;font-weight:bold;">if</span> <span style="color:#ffffff;font-weight:bold;">userid_exist</span><span style="color:#fbfbfb;">(</span>userid<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">237 </span>          <span style="color:#5ad5d5;font-weight:bold;">print</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">"该用户貌似已经检索"</span><span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">238 </span>          <span style="color:#5ad5d5;font-weight:bold;">return</span>
<span style="color:#e7e77f;">239 </span>       <span style="color:#ffffff;font-weight:bold;">create_dir</span><span style="color:#fbfbfb;">(</span>userid<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">240 </span>       
<span style="color:#e7e77f;">241 </span>       file_name <span style="color:#fbfbfb;">=</span> <span style="color:#ffffff;font-weight:bold;">get_file_name</span><span style="color:#fbfbfb;">(</span>userid<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">242 </span>       <span style="color:#d174a8;font-weight:bold;">file</span> <span style="color:#fbfbfb;">=</span> <span style="color:#d174a8;font-weight:bold;">open</span><span style="color:#fbfbfb;">(</span>file_name<span style="color:#fbfbfb;">,</span> <span style="color:#3fffa7;">'wb'</span><span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">243 </span>       
<span style="color:#e7e77f;">244 </span>       ff <span style="color:#fbfbfb;">=</span> <span style="color:#ffffff;font-weight:bold;">Fetcher</span><span style="color:#fbfbfb;">(</span>manager<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">245 </span>       ff<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">push</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'http://www.qiushibaike.com/users/'</span><span style="color:#fbfbfb;">+</span>userid<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">246 </span>       req<span style="color:#fbfbfb;">,</span>ans <span style="color:#fbfbfb;">=</span> ff<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">pop</span><span style="color:#fbfbfb;">()</span>
<span style="color:#e7e77f;">247 </span>       
<span style="color:#e7e77f;">248 </span>       soup <span style="color:#fbfbfb;">=</span> <span style="color:#ffffff;font-weight:bold;">BeautifulSoup</span><span style="color:#fbfbfb;">(</span>ans<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">decode</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'utf-8'</span><span style="color:#fbfbfb;">),</span><span style="color:#3fffa7;">"html.parser"</span><span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">249 </span>       user_name<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">""</span>
<span style="color:#e7e77f;">250 </span>       <span style="color:#5ad5d5;font-weight:bold;">for</span> ii <span style="color:#5ad5d5;font-weight:bold;">in</span> soup<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">find_all</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'span'</span><span style="color:#fbfbfb;">,</span>class_<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">"user_center"</span><span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">251 </span>           user_name <span style="color:#fbfbfb;">=</span> ii<span style="color:#fbfbfb;">.</span>text
<span style="color:#e7e77f;">252 </span>       <span style="color:#5ad5d5;font-weight:bold;">if not</span> user_name<span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">253 </span>           <span style="color:#5ad5d5;font-weight:bold;">del</span> ff
<span style="color:#e7e77f;">254 </span>           <span style="color:#d174a8;font-weight:bold;">file</span><span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">close</span><span style="color:#fbfbfb;">()</span>
<span style="color:#e7e77f;">255 </span>           <span style="color:#5ad5d5;font-weight:bold;">return</span>
<span style="color:#e7e77f;">256 </span>       
<span style="color:#e7e77f;">257 </span>       <span style="color:#e7e77f;">#把第一页的写进文件</span>
<span style="color:#e7e77f;">258 </span>       count <span style="color:#fbfbfb;">=</span> <span style="color:#ffffff;font-weight:bold;">write_file</span><span style="color:#fbfbfb;">(</span><span style="color:#d174a8;font-weight:bold;">file</span><span style="color:#fbfbfb;">,</span>soup<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">259 </span>       <span style="color:#5ad5d5;font-weight:bold;">print</span><span style="color:#fbfbfb;">(</span>user_name<span style="color:#fbfbfb;">+</span><span style="color:#3fffa7;">" "</span><span style="color:#fbfbfb;">+</span><span style="color:#d174a8;font-weight:bold;">str</span><span style="color:#fbfbfb;">(</span>count<span style="color:#fbfbfb;">)+</span><span style="color:#3fffa7;">"条糗事 [http://www.qiushibaike.com/users/"</span><span style="color:#fbfbfb;">+</span>userid<span style="color:#fbfbfb;">+</span><span style="color:#3fffa7;">"/articles/page/1]"</span><span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">260 </span>
<span style="color:#e7e77f;">261 </span>       <span style="color:#e7e77f;">#把余下的页面请求完</span>
<span style="color:#e7e77f;">262 </span>       max_page <span style="color:#fbfbfb;">=</span> <span style="color:#ffffff;font-weight:bold;">get_max_page</span><span style="color:#fbfbfb;">(</span>soup<span style="color:#fbfbfb;">)+</span><span style="color:#3fffa7;">1</span>
<span style="color:#e7e77f;">263 </span>       <span style="color:#5ad5d5;font-weight:bold;">for</span> i <span style="color:#5ad5d5;font-weight:bold;">in</span> <span style="color:#d174a8;font-weight:bold;">range</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">2</span><span style="color:#fbfbfb;">,</span>max_page<span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">264 </span>           <span style="color:#e7e77f;">#print("加入列表 [http://www.qiushibaike.com/users/"+userid+"/articles/page/"+str(i)+"]")</span>
<span style="color:#e7e77f;">265 </span>           ff<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">push</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">"http://www.qiushibaike.com/users/"</span><span style="color:#fbfbfb;">+</span>userid<span style="color:#fbfbfb;">+</span><span style="color:#3fffa7;">"/articles/page/"</span><span style="color:#fbfbfb;">+</span><span style="color:#d174a8;font-weight:bold;">str</span><span style="color:#fbfbfb;">(</span>i<span style="color:#fbfbfb;">))</span>
<span style="color:#e7e77f;">266 </span>
<span style="color:#e7e77f;">267 </span>       <span style="color:#5ad5d5;font-weight:bold;">while</span> ff<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">taskleft</span><span style="color:#fbfbfb;">():</span>
<span style="color:#e7e77f;">268 </span>           req<span style="color:#fbfbfb;">,</span>ans <span style="color:#fbfbfb;">=</span> ff<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">pop</span><span style="color:#fbfbfb;">()</span>
<span style="color:#e7e77f;">269 </span>           soup <span style="color:#fbfbfb;">=</span> <span style="color:#ffffff;font-weight:bold;">BeautifulSoup</span><span style="color:#fbfbfb;">(</span>ans<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">decode</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'utf-8'</span><span style="color:#fbfbfb;">),</span><span style="color:#3fffa7;">"html.parser"</span><span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">270 </span>           count <span style="color:#fbfbfb;">=</span> <span style="color:#ffffff;font-weight:bold;">write_file</span><span style="color:#fbfbfb;">(</span><span style="color:#d174a8;font-weight:bold;">file</span><span style="color:#fbfbfb;">,</span>soup<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">271 </span>           <span style="color:#5ad5d5;font-weight:bold;">print</span><span style="color:#fbfbfb;">(</span>user_name<span style="color:#fbfbfb;">+</span><span style="color:#3fffa7;">" "</span><span style="color:#fbfbfb;">+</span><span style="color:#d174a8;font-weight:bold;">str</span><span style="color:#fbfbfb;">(</span>count<span style="color:#fbfbfb;">)+</span><span style="color:#3fffa7;">"条糗事 ["</span><span style="color:#fbfbfb;">+</span>req<span style="color:#fbfbfb;">+</span><span style="color:#3fffa7;">"]"</span><span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">272 </span>
<span style="color:#e7e77f;">273 </span>       <span style="color:#5ad5d5;font-weight:bold;">print</span><span style="color:#fbfbfb;">(</span>user_name<span style="color:#fbfbfb;">+</span><span style="color:#3fffa7;">" 的线程资源被删除"</span><span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">274 </span>       <span style="color:#5ad5d5;font-weight:bold;">del</span> ff
<span style="color:#e7e77f;">275 </span>       <span style="color:#d174a8;font-weight:bold;">file</span><span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">close</span><span style="color:#fbfbfb;">()</span>
<span style="color:#e7e77f;">276 </span>       <span style="color:#5ad5d5;font-weight:bold;">return</span>
<span style="color:#e7e77f;">277 </span>
<span style="color:#e7e77f;">278 </span>       
<span style="color:#e7e77f;">279 </span>
<span style="color:#e7e77f;">280 </span><span style="color:#5ad5d5;font-weight:bold;">def</span> <span style="color:#ffffff;font-weight:bold;">main</span><span style="color:#fbfbfb;">():</span>
<span style="color:#e7e77f;">281 </span>       <span style="color:#e7e77f;">#os.mkdir('qiushibaike')</span>
<span style="color:#e7e77f;">282 </span>       <span style="color:#e7e77f;">#store_this_user("13843355")</span>
<span style="color:#e7e77f;">283 </span>
<span style="color:#e7e77f;">284 </span>       opener <span style="color:#fbfbfb;">=</span> urllib<span style="color:#fbfbfb;">.</span>request<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">build_opener</span><span style="color:#fbfbfb;">(</span>urllib<span style="color:#fbfbfb;">.</span>request<span style="color:#fbfbfb;">.</span>HTTPHandler<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">285 </span>       opener<span style="color:#fbfbfb;">.</span>addheaders <span style="color:#fbfbfb;">= [(</span><span style="color:#3fffa7;">'User-agent'</span><span style="color:#fbfbfb;">,</span> <span style="color:#3fffa7;">'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11'</span><span style="color:#fbfbfb;">)]</span>
<span style="color:#e7e77f;">286 </span>       
<span style="color:#e7e77f;">287 </span>
<span style="color:#e7e77f;">288 </span>       worker_manager <span style="color:#fbfbfb;">=</span> <span style="color:#ffffff;font-weight:bold;">WorkerManager</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">128</span><span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">289 </span>       <span style="color:#e7e77f;">#manager = WorkerManager(128)</span>
<span style="color:#e7e77f;">290 </span>
<span style="color:#e7e77f;">291 </span>       <span style="color:#e7e77f;">#ff = Fetcher(worker_manager)</span>
<span style="color:#e7e77f;">292 </span>       next_link <span style="color:#fbfbfb;">=</span> <span style="color:#3fffa7;">'http://www.qiushibaike.com'</span>
<span style="color:#e7e77f;">293 </span>       <span style="color:#e7e77f;">#ff.push(next_link)</span>
<span style="color:#e7e77f;">294 </span>       
<span style="color:#e7e77f;">295 </span>
<span style="color:#e7e77f;">296 </span>       page_num<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">0</span>
<span style="color:#e7e77f;">297 </span>       
<span style="color:#e7e77f;">298 </span>       <span style="color:#5ad5d5;font-weight:bold;">while True</span><span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">299 </span>           page_num <span style="color:#fbfbfb;">+=</span> <span style="color:#3fffa7;">1</span>
<span style="color:#e7e77f;">300 </span>           <span style="color:#5ad5d5;font-weight:bold;">print</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">"自动翻到第"</span><span style="color:#fbfbfb;">+</span><span style="color:#d174a8;font-weight:bold;">str</span><span style="color:#fbfbfb;">(</span>page_num<span style="color:#fbfbfb;">)+</span><span style="color:#3fffa7;">"页 "</span> <span style="color:#fbfbfb;">+</span> next_link<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">301 </span>           ans <span style="color:#fbfbfb;">=</span> opener<span style="color:#fbfbfb;">.</span><span style="color:#d174a8;font-weight:bold;">open</span><span style="color:#fbfbfb;">(</span>next_link<span style="color:#fbfbfb;">).</span><span style="color:#ffffff;font-weight:bold;">read</span><span style="color:#fbfbfb;">()</span>
<span style="color:#e7e77f;">302 </span>           next_link<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">""</span>
<span style="color:#e7e77f;">303 </span>           <span style="color:#e7e77f;">#req,ans = ff.pop()</span>
<span style="color:#e7e77f;">304 </span>           <span style="color:#e7e77f;">#print("adfsdfsdfsdfds")</span>
<span style="color:#e7e77f;">305 </span>           soup <span style="color:#fbfbfb;">=</span> <span style="color:#ffffff;font-weight:bold;">BeautifulSoup</span><span style="color:#fbfbfb;">(</span>ans<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">decode</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'utf-8'</span><span style="color:#fbfbfb;">),</span><span style="color:#3fffa7;">"html.parser"</span><span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">306 </span>
<span style="color:#e7e77f;">307 </span>           <span style="color:#5ad5d5;font-weight:bold;">for</span> ii <span style="color:#5ad5d5;font-weight:bold;">in</span> soup<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">find_all</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'a'</span><span style="color:#fbfbfb;">,</span>class_<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">"next"</span><span style="color:#fbfbfb;">,</span>text<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">"下一页"</span><span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">308 </span>               next_link <span style="color:#fbfbfb;">=</span> ii<span style="color:#fbfbfb;">[</span><span style="color:#3fffa7;">"href"</span><span style="color:#fbfbfb;">]</span>
<span style="color:#e7e77f;">309 </span>           <span style="color:#5ad5d5;font-weight:bold;">if not</span> next_link<span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">310 </span>               <span style="color:#5ad5d5;font-weight:bold;">print</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">"what the fuck!!!"</span><span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">311 </span>               <span style="color:#5ad5d5;font-weight:bold;">break</span>
<span style="color:#e7e77f;">312 </span>           next_link <span style="color:#fbfbfb;">=</span> <span style="color:#3fffa7;">"http://www.qiushibaike.com"</span><span style="color:#fbfbfb;">+</span>next_link
<span style="color:#e7e77f;">313 </span>           <span style="color:#e7e77f;">#ff.push(next_link)</span>
<span style="color:#e7e77f;">314 </span>
<span style="color:#e7e77f;">315 </span>           <span style="color:#5ad5d5;font-weight:bold;">for</span> ii <span style="color:#5ad5d5;font-weight:bold;">in</span> soup<span style="color:#fbfbfb;">.</span><span style="color:#ffffff;font-weight:bold;">find_all</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'div'</span><span style="color:#fbfbfb;">,</span>class_<span style="color:#fbfbfb;">=</span><span style="color:#3fffa7;">"author"</span><span style="color:#fbfbfb;">):</span>
<span style="color:#e7e77f;">316 </span>               <span style="color:#5ad5d5;font-weight:bold;">print</span><span style="color:#fbfbfb;">(</span>ii<span style="color:#fbfbfb;">.</span>a<span style="color:#fbfbfb;">[</span><span style="color:#3fffa7;">"href"</span><span style="color:#fbfbfb;">].</span><span style="color:#ffffff;font-weight:bold;">split</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'/'</span><span style="color:#fbfbfb;">)[</span><span style="color:#3fffa7;">2</span><span style="color:#fbfbfb;">])</span>
<span style="color:#e7e77f;">317 </span>               <span style="color:#ffffff;font-weight:bold;">store_this_user</span><span style="color:#fbfbfb;">(</span>ii<span style="color:#fbfbfb;">.</span>a<span style="color:#fbfbfb;">[</span><span style="color:#3fffa7;">"href"</span><span style="color:#fbfbfb;">].</span><span style="color:#ffffff;font-weight:bold;">split</span><span style="color:#fbfbfb;">(</span><span style="color:#3fffa7;">'/'</span><span style="color:#fbfbfb;">)[</span><span style="color:#3fffa7;">2</span><span style="color:#fbfbfb;">],</span> worker_manager<span style="color:#fbfbfb;">)</span>
<span style="color:#e7e77f;">318 </span>       
<span style="color:#e7e77f;">319 </span>
<span style="color:#e7e77f;">320 </span>
<span style="color:#e7e77f;">321 </span><span style="color:#3fffa7;">'''</span>
<span style="color:#e7e77f;">322 </span>
<span style="color:#e7e77f;">323 </span><span style="color:#3fffa7;">       file.close()</span>
<span style="color:#e7e77f;">324 </span>
<span style="color:#e7e77f;">325 </span>
<span style="color:#e7e77f;">326 </span><span style="color:#3fffa7;">       ff = Fetcher(10)</span>
<span style="color:#e7e77f;">327 </span><span style="color:#3fffa7;">       ff.push('http://www.qiushibaike.com/users/14870461')</span>
<span style="color:#e7e77f;">328 </span><span style="color:#3fffa7;">       req,ans = ff.pop()</span>
<span style="color:#e7e77f;">329 </span>       
<span style="color:#e7e77f;">330 </span><span style="color:#3fffa7;">       print(ans.decode('utf8'))</span>
<span style="color:#e7e77f;">331 </span>       
<span style="color:#e7e77f;">332 </span>
<span style="color:#e7e77f;">333 </span><span style="color:#3fffa7;">       #os.system("pause")</span>
<span style="color:#e7e77f;">334 </span>    
<span style="color:#e7e77f;">335 </span>
<span style="color:#e7e77f;">336 </span>
<span style="color:#e7e77f;">337 </span><span style="color:#3fffa7;">       testgbk='汉字'</span>
<span style="color:#e7e77f;">338 </span><span style="color:#3fffa7;">       testunit=testgbk.encode('gbk')    #--汉字解码</span>
<span style="color:#e7e77f;">339 </span><span style="color:#3fffa7;">       print(testunit)</span>
<span style="color:#e7e77f;">340 </span>       
<span style="color:#e7e77f;">341 </span><span style="color:#3fffa7;">       testutf8=testgbk.encode('utf-8')  #--转utf-8编码</span>
<span style="color:#e7e77f;">342 </span><span style="color:#3fffa7;">       print(testutf8)</span>
<span style="color:#e7e77f;">343 </span>       
<span style="color:#e7e77f;">344 </span><span style="color:#3fffa7;">       testunit=testutf8.decode('utf-8')  #--utf-8解码</span>
<span style="color:#e7e77f;">345 </span><span style="color:#3fffa7;">       print(testunit)</span>
<span style="color:#e7e77f;">346 </span>       
<span style="color:#e7e77f;">347 </span><span style="color:#3fffa7;">       testgbk=testunit.encode('gbk')    #--转gbk编码</span>
<span style="color:#e7e77f;">348 </span><span style="color:#3fffa7;">       print(testgbk)</span>
<span style="color:#e7e77f;">349 </span><span style="color:#3fffa7;">'''</span>
<span style="color:#e7e77f;">350 </span>
<span style="color:#e7e77f;">351 </span>
<span style="color:#e7e77f;">352 </span>       
<span style="color:#e7e77f;">353 </span><span style="color:#3fffa7;">'''</span>
<span style="color:#e7e77f;">354 </span><span style="color:#3fffa7;">       links = ['http://item.jd.com/</span><span style="color:#bfbfef;">%d</span><span style="color:#3fffa7;">.html'</span><span style="color:#bfbfef;">%i</span> <span style="color:#3fffa7;">for i in range(1746854,1746860)]</span>
<span style="color:#e7e77f;">355 </span><span style="color:#3fffa7;">       ff = Fetcher(10)</span>
<span style="color:#e7e77f;">356 </span><span style="color:#3fffa7;">       for url in links:</span>
<span style="color:#e7e77f;">357 </span><span style="color:#3fffa7;">           ff.push(url)</span>
<span style="color:#e7e77f;">358 </span>
<span style="color:#e7e77f;">359 </span><span style="color:#3fffa7;">       while ff.taskleft():</span>
<span style="color:#e7e77f;">360 </span><span style="color:#3fffa7;">           (url,content) = ff.pop()</span>
<span style="color:#e7e77f;">361 </span><span style="color:#3fffa7;">           print(url,len(content))</span>
<span style="color:#e7e77f;">362 </span><span style="color:#3fffa7;">'''</span>
<span style="color:#e7e77f;">363 </span>
<span style="color:#e7e77f;">364 </span>       
<span style="color:#e7e77f;">365 </span>
<span style="color:#e7e77f;">366 </span><span style="color:#3fffa7;">'''</span>
<span style="color:#e7e77f;">367 </span><span style="color:#3fffa7;">url = 'http://www.sina.com'</span>
<span style="color:#e7e77f;">368 </span><span style="color:#3fffa7;">user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)' </span>
<span style="color:#e7e77f;">369 </span><span style="color:#3fffa7;">values = {'name' : 'Michael Foord', </span>
<span style="color:#e7e77f;">370 </span><span style="color:#3fffa7;">             'location' : 'pythontab', </span>
<span style="color:#e7e77f;">371 </span><span style="color:#3fffa7;">             'language' : 'Python' }</span>
<span style="color:#e7e77f;">372 </span><span style="color:#3fffa7;">headers = { 'User-Agent' : user_agent }</span>
<span style="color:#e7e77f;">373 </span>
<span style="color:#e7e77f;">374 </span><span style="color:#3fffa7;">data = urllib.parse.urlencode(values) </span>
<span style="color:#e7e77f;">375 </span><span style="color:#3fffa7;">#req = urllib.request.Request(url, data, headers)</span>
<span style="color:#e7e77f;">376 </span><span style="color:#3fffa7;">req = urllib.request.Request('http://www.baidu.com')</span>
<span style="color:#e7e77f;">377 </span><span style="color:#3fffa7;">req.add_header('User-Agent', 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)')</span>
<span style="color:#e7e77f;">378 </span><span style="color:#3fffa7;">'''</span>
<span style="color:#e7e77f;">379 </span>
<span style="color:#e7e77f;">380 </span>
<span style="color:#e7e77f;">381 </span><span style="color:#3fffa7;">'''</span>
<span style="color:#e7e77f;">382 </span><span style="color:#3fffa7;">##################################</span>
<span style="color:#e7e77f;">383 </span><span style="color:#3fffa7;">def cbk(a, b, c):</span>
<span style="color:#e7e77f;">384 </span>       
<span style="color:#e7e77f;">385 </span><span style="color:#3fffa7;">       #回调函数</span>
<span style="color:#e7e77f;">386 </span><span style="color:#3fffa7;">       #@a: 已经下载的数据块</span>
<span style="color:#e7e77f;">387 </span><span style="color:#3fffa7;">       #@b: 数据块的大小</span>
<span style="color:#e7e77f;">388 </span><span style="color:#3fffa7;">       #@c: 远程文件的大小</span>
<span style="color:#e7e77f;">389 </span>       
<span style="color:#e7e77f;">390 </span><span style="color:#3fffa7;">       per = 100.0 * a * b / c</span>
<span style="color:#e7e77f;">391 </span><span style="color:#3fffa7;">       if per &gt; 100:</span>
<span style="color:#e7e77f;">392 </span><span style="color:#3fffa7;">           per = 100</span>
<span style="color:#e7e77f;">393 </span><span style="color:#3fffa7;">       num = int(per)</span>
<span style="color:#e7e77f;">394 </span>       
<span style="color:#e7e77f;">395 </span>    
<span style="color:#e7e77f;">396 </span><span style="color:#3fffa7;">       print('[',end='')</span>
<span style="color:#e7e77f;">397 </span><span style="color:#3fffa7;">       for i in range(num):</span>
<span style="color:#e7e77f;">398 </span><span style="color:#3fffa7;">           print('#',end='')</span>
<span style="color:#e7e77f;">399 </span><span style="color:#3fffa7;">       print('%.2f]'</span> <span style="color:#bfbfef;">%(per)</span><span style="color:#3fffa7;">, end='')</span>
<span style="color:#e7e77f;">400 </span>
<span style="color:#e7e77f;">401 </span><span style="color:#3fffa7;">url = 'http://www.sina.com.cn'</span>
<span style="color:#e7e77f;">402 </span><span style="color:#3fffa7;">local = 'e:</span><span style="color:#bfbfef;">\</span><span style="color:#3fffa7;">sina.html'</span>
<span style="color:#e7e77f;">403 </span><span style="color:#3fffa7;">urllib.request.urlretrieve(url, local, cbk)</span>
<span style="color:#e7e77f;">404 </span>
<span style="color:#e7e77f;">405 </span><span style="color:#3fffa7;">input()</span>
<span style="color:#e7e77f;">406 </span><span style="color:#3fffa7;">os.system("pause")</span>
<span style="color:#e7e77f;">407 </span><span style="color:#3fffa7;">##################################</span>
<span style="color:#e7e77f;">408 </span><span style="color:#3fffa7;">'''</span>
<span style="color:#e7e77f;">409 </span>
<span style="color:#e7e77f;">410 </span><span style="color:#3fffa7;">'''</span>
<span style="color:#e7e77f;">411 </span><span style="color:#3fffa7;">try:</span>
<span style="color:#e7e77f;">412 </span><span style="color:#3fffa7;">       response = urllib.request.urlopen(req)</span>
<span style="color:#e7e77f;">413 </span><span style="color:#3fffa7;">       print('ffdfsdfsf')</span>
<span style="color:#e7e77f;">414 </span><span style="color:#3fffa7;">except urllib.error.URLError as e:</span>
<span style="color:#e7e77f;">415 </span><span style="color:#3fffa7;">       if hasattr(e, 'reason'):</span>
<span style="color:#e7e77f;">416 </span><span style="color:#3fffa7;">           print('We failed to reach a server.')</span>
<span style="color:#e7e77f;">417 </span><span style="color:#3fffa7;">           print('Reason: ', e.reason)</span>
<span style="color:#e7e77f;">418 </span><span style="color:#3fffa7;">       elif hasattr(e, 'code'):</span>
<span style="color:#e7e77f;">419 </span><span style="color:#3fffa7;">           print('The server cannot fulfill the request.')</span>
<span style="color:#e7e77f;">420 </span><span style="color:#3fffa7;">           print('Reason: ', e.code)</span>
<span style="color:#e7e77f;">421 </span><span style="color:#3fffa7;">else:</span>
<span style="color:#e7e77f;">422 </span><span style="color:#3fffa7;">           #print(response.info)</span>
<span style="color:#e7e77f;">423 </span><span style="color:#3fffa7;">           #print(response.getcode())</span>
<span style="color:#e7e77f;">424 </span><span style="color:#3fffa7;">           response_context = response.read()</span>
<span style="color:#e7e77f;">425 </span><span style="color:#3fffa7;">           print(response_context.decode("utf8"))</span>
<span style="color:#e7e77f;">426 </span><span style="color:#3fffa7;">'''</span>
<span style="color:#e7e77f;">427 </span>
<span style="color:#e7e77f;">428 </span>
<span style="color:#e7e77f;">429 </span><span style="color:#5ad5d5;font-weight:bold;">if</span> __name__<span style="color:#fbfbfb;">==</span><span style="color:#3fffa7;">"__main__"</span><span style="color:#fbfbfb;">:</span>
<span style="color:#e7e77f;">430 </span>       <span style="color:#ffffff;font-weight:bold;">main</span><span style="color:#fbfbfb;">()</span>

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

1

2 import urllib.request

3 import urllib.parse

4 import time

5 import os

6 import threading

7 import queue

8 import bs4

9 from bs4 import BeautifulSoup

10 import shutil

11 import errno

12 import sys

13

14 headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',

15 'Accept':'text/html;q=0.9,*/*;q=0.8',

16 'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.3',

17 'Accept-Encoding':'gzip',

18 'Connection':'close',

19 'Referer':None #注意如果依然不能抓取，这里可以设置抓取网站的host

20 }

21

22 #############################################################

23 # working thread

24 class Worker(threading.Thread):

25 worker_count = 0

26

27 def __init__( self, workQueue, resultQueue, timeout = 0, **kwds):

28 threading.Thread.__init__( self, **kwds )

29 self.id = Worker.worker_count

30 Worker.worker_count += 1

31 self.setDaemon( True )

32 self.workQueue = workQueue

33 self.resultQueue = resultQueue

34 self.timeout = timeout

35 self.start()

36

37 def run( self ):

38 ''' the get-some-work, do-some-work main loop of worker threads '''

39 while True:

40 try:

41 callable, args, kwds = self.workQueue.get()

42 res = callable(*args, **kwds)

43 #print("worker[%2d]: %s" % (self.id, str(res) ))

44 #self.resultQueue.put( res )

45 except queue.Empty:

46 pass

47 #break

48 except:

49 print('worker[%2d]' % self.id, sys.exc_info()[:2])

50

51 class WorkerManager:

52 def __init__(self, num_of_workers=10, timeout = 1):

53 self.workQueue = queue.Queue()

54 self.resultQueue = queue.Queue()

55 self.workers = []

56 self.timeout = timeout

57 self._recruitThreads(num_of_workers)

58

59 def _recruitThreads(self, num_of_workers):

60 for i in range(num_of_workers):

61 worker = Worker(self.workQueue, self.resultQueue, self.timeout)

62 self.workers.append(worker)

63

64 def wait_for_complete(self):

65 # ...then, wait for each of them to terminate:

66 while len(self.workers):

67 worker = self.workers.pop()

68 worker.join()

69 if worker.isAlive() and not self.workQueue.empty():

70 self.workers.append( worker )

71 print("All jobs are are completed.")

72

73 def add_job( self, callable, *args, **kwds ):

74 self.workQueue.put( (callable, args, kwds) )

75

76 def get_result( self, *args, **kwds ):

77 return self.resultQueue.get( *args, **kwds )

78

79

80

81 #############################################################

82

83

84 class Fetcher:

85 def __init__(self,manager):

86 self.opener = urllib.request.build_opener(urllib.request.HTTPHandler)

87 self.opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11')]

88 self.lock = threading.Lock() #线程锁

89 self.q_req = queue.Queue() #任务队列

90 self.q_ans = queue.Queue() #完成队列

91 self.__q_retry = queue.Queue() #重试队列

92 #self.threads_num = threads_num + 1 # 1代表了重试线程

93 #self.__threads = []

94

95 #重试线程

96 manager.add_job(self.threadretry,self)

97 #self.__retry_thread = threading.Thread(target=self.threadretry)

98 #self.__threads.append(self.__retry_thread)

99 #self.__retry_thread.setDaemon(True)

100 #self.__retry_thread.start()

101

102 #开启@threads_num个工作线程

103 manager.add_job(self.threadget,self)

104 #for i in range(threads_num):

105 # t = threading.Thread(target=self.threadget)

106 # self.__threads.append(t)

107 # t.setDaemon(True)

108 # t.start()

109

110 self.running = 0

111

112 #def __del__(self): #解构时需等待两个队列完成

113 #print("will delete")

114 #self.q_req.join()

115 #self.q_ans.join()

116 #self.__q_retry.join()

117 #print("deleted")

118 #for i in range(self.threads_num+1):

119 # self.__threads[i].exit()

120 #print("还有"+str(threading.active_count())+"个活跃线程")

121

122 def taskleft(self):

123 return self.q_req.qsize()+self.q_ans.qsize()+self.running

124

125 def push(self,req):

126 self.q_req.put(req)

127

128 def pop(self):

129 return self.q_ans.get()

130

131 def threadretry(self,_self):

132 while True:

133 try:

134 req = _self.__q_retry.get(timeout=1) #get(self,block=True,timeout=None)

135 except queue.Empty:

136 break

137

138 #with _self.lock:

139 _self.running += 1

140

141 try:

142 ans = _self.opener.open(req).read()

143

144 except urllib.error.URLError as e:

145 ans = ''

146 if hasattr(e, 'reason'):

147 print('We failed to reach a server.')

148 print('Reason: ', e.reason)

149 elif hasattr(e, 'code'):

150 print('The server cannot fulfill the request.')

151 print('Reason: ', e.code)

152 else:

153 if ans:

154 _self.q_ans.put((req,ans))

155 finally:

156 #with _self.lock:

157 _self.running -= 1

158

159 _self.__q_retry.task_done()

160

161

162

163 def threadget(self,_self):

164 while True:

165 try:

166 req = _self.q_req.get(timeout=1)

167 except queue.Empty:

168 break

169 #with _self.lock: #要保证该操作的原子性，进入critical area

170 _self.running += 1

171

172 try:

173 ans = _self.opener.open(req).read()

174 except urllib.error.URLError as e:

175 ans = ''

176 if hasattr(e, 'reason'):

177 print('We failed to reach a server.')

178 print('Reason: ', e.reason)

179 _self.__q_retry.put(req)

180 elif hasattr(e, 'code'):

181 print('The server cannot fulfill the request.')

182 print('Reason: ', e.code)

183 _self.__q_retry.put(req)

184 else:

185 if ans:

186 _self.q_ans.put((req,ans))

187 else:

188 _self.__q_retry.put(req)

189 finally:

190 #with _self.lock:

191 _self.running -= 1

192 _self.q_req.task_done()

193

194

195 def create_dir(userid,domain='qiushibaike'):

196 dir_name = domain + '/' + userid

197 try:

198 os.mkdir(dir_name)

199 except OSError as e:

200 if e.errno == errno.EEXIST and os.path.isdir(dir_name):

201 pass

202 else:

203 print(str(e))

204

205 def userid_exist(userid):

206 return os.path.isdir('qiushibaike'+'/'+userid)

207

208 def get_file_name(userid):

209 current_time = time.strftime("%Y-%m-%d",time.localtime())

210 return 'qiushibaike'+'/'+userid+'/'+current_time+'.txt'

211

212

213

214 def write_file(file,soup):

215 count=0

216 for ii in soup.find_all("div",class_="content clearfix"):

217 #print(ii.a["href"])

218 #print(ii.a.text)

219 if ii.a.text:

220 count += 1

221 file.write(bytes(ii.a["href"],encoding="utf-8"))

222 file.write(bytes('rn',encoding="utf-8"))

223 file.write(bytes(ii.a.text,encoding="utf-8"))

224 file.write(bytes("rnrn",encoding="utf-8"))

225 return count

226

227 def get_max_page(soup):

228 #ii=bs4.element.Tag()

229 num=0

230 for jj in soup.find_all('a',rel="next",class_=None):

231 num=int(jj.text)

232 return num

233

234

235 def store_this_user(userid,manager):

236 if userid_exist(userid):

237 print("该用户貌似已经检索")

238 return

239 create_dir(userid)

240

241 file_name = get_file_name(userid)

242 file = open(file_name, 'wb')

243

244 ff = Fetcher(manager)

245 ff.push('http://www.qiushibaike.com/users/'+userid)

246 req,ans = ff.pop()

247

248 soup = BeautifulSoup(ans.decode('utf-8'),"html.parser")

249 user_name=""

250 for ii in soup.find_all('span',class_="user_center"):

251 user_name = ii.text

252 if not user_name:

253 del ff

254 file.close()

255 return

256

257 #把第一页的写进文件

258 count = write_file(file,soup)

259 print(user_name+" "+str(count)+"条糗事 [http://www.qiushibaike.com/users/"+userid+"/articles/page/1]")

260

261 #把余下的页面请求完

262 max_page = get_max_page(soup)+1

263 for i in range(2,max_page):

264 #print("加入列表 [http://www.qiushibaike.com/users/"+userid+"/articles/page/"+str(i)+"]")

265 ff.push("http://www.qiushibaike.com/users/"+userid+"/articles/page/"+str(i))

266

267 while ff.taskleft():

268 req,ans = ff.pop()

269 soup = BeautifulSoup(ans.decode('utf-8'),"html.parser")

270 count = write_file(file,soup)

271 print(user_name+" "+str(count)+"条糗事 ["+req+"]")

272

273 print(user_name+" 的线程资源被删除")

274 del ff

275 file.close()

276 return

277

278

279

280 def main():

281 #os.mkdir('qiushibaike')

282 #store_this_user("13843355")

283

284 opener = urllib.request.build_opener(urllib.request.HTTPHandler)

285 opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11')]

286

287

288 worker_manager = WorkerManager(128)

289 #manager = WorkerManager(128)

290

291 #ff = Fetcher(worker_manager)

292 next_link = 'http://www.qiushibaike.com'

293 #ff.push(next_link)

294

295

296 page_num=0

297

298 while True:

299 page_num += 1

300 print("自动翻到第"+str(page_num)+"页 " + next_link)

301 ans = opener.open(next_link).read()

302 next_link=""

303 #req,ans = ff.pop()

304 #print("adfsdfsdfsdfds")

305 soup = BeautifulSoup(ans.decode('utf-8'),"html.parser")

306

307 for ii in soup.find_all('a',class_="next",text="下一页"):

308 next_link = ii["href"]

309 if not next_link:

310 print("what the fuck!!!")

311 break

312 next_link = "http://www.qiushibaike.com"+next_link

313 #ff.push(next_link)

314

315 for ii in soup.find_all('div',class_="author"):

316 print(ii.a["href"].split('/')[2])

317 store_this_user(ii.a["href"].split('/')[2], worker_manager)

318

319

320

321 '''

322

323 file.close()

324

325

326 ff = Fetcher(10)

327 ff.push('http://www.qiushibaike.com/users/14870461')

328 req,ans = ff.pop()

329

330 print(ans.decode('utf8'))

331

332

333 #os.system("pause")

334

335

336

337 testgbk='汉字'

338 testunit=testgbk.encode('gbk') #--汉字解码

339 print(testunit)

340

341 testutf8=testgbk.encode('utf-8') #--转utf-8编码

342 print(testutf8)

343

344 testunit=testutf8.decode('utf-8') #--utf-8解码

345 print(testunit)

346

347 testgbk=testunit.encode('gbk') #--转gbk编码

348 print(testgbk)

349 '''

350

351

352

353 '''

354 links = ['http://item.jd.com/%d.html'%i for i in range(1746854,1746860)]

355 ff = Fetcher(10)

356 for url in links:

357 ff.push(url)

358

359 while ff.taskleft():

360 (url,content) = ff.pop()

361 print(url,len(content))

362 '''

363

364

365

366 '''

367 url = 'http://www.sina.com'

368 user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'

369 values = {'name' : 'Michael Foord',

370 'location' : 'pythontab',

371 'language' : 'Python' }

372 headers = { 'User-Agent' : user_agent }

373

374 data = urllib.parse.urlencode(values)

375 #req = urllib.request.Request(url, data, headers)

376 req = urllib.request.Request('http://www.baidu.com')

377 req.add_header('User-Agent', 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)')

378 '''

379

380

381 '''

382 ##################################

383 def cbk(a, b, c):

384

385 #回调函数

386 #@a: 已经下载的数据块

387 #@b: 数据块的大小

388 #@c: 远程文件的大小

389

390 per = 100.0 * a * b / c

391 if per > 100:

392 per = 100

393 num = int(per)

394

395

396 print('[',end='')

397 for i in range(num):

398 print('#',end='')

399 print('%.2f]' %(per), end='')

400

401 url = 'http://www.sina.com.cn'

402 local = 'e:\sina.html'

403 urllib.request.urlretrieve(url, local, cbk)

404

405 input()

406 os.system("pause")

407 ##################################

408 '''

409

410 '''

411 try:

412 response = urllib.request.urlopen(req)

413 print('ffdfsdfsf')

414 except urllib.error.URLError as e:

415 if hasattr(e, 'reason'):

416 print('We failed to reach a server.')

417 print('Reason: ', e.reason)

418 elif hasattr(e, 'code'):

419 print('The server cannot fulfill the request.')

420 print('Reason: ', e.code)

421 else:

422 #print(response.info)

423 #print(response.getcode())

424 response_context = response.read()

425 print(response_context.decode("utf8"))

426 '''

427

428

429 if __name__=="__main__":

430 main()

如无特殊说明，文章均为本站原创，转载请注明出处
源自: 王明军的博客
本文链接地址: Python简易爬虫 – 抓取糗事百科糗事

王明军的博客

一个程序员的杂货铺

Python简易爬虫 – 抓取糗事百科糗事

赞过：

相关

发表评论取消回复

Python简易爬虫 – 抓取糗事百科糗事

SHARE TO

赞过：

相关

发表评论 取消回复

发表评论取消回复