OPT爬虫

受到      [OPT] 简单python爬虫的source code分享    启发，我开发了一个OPT case status爬虫， 用了很多之前帖子里面的code，加了multi threading，速度提高很多，自己加一下url_prefix，我不能发链接

1. from __future__ import division
   
2. from bs4 import BeautifulSoup
   
3. import requests
   
4. import thread
   
5. from time import sleep
   
6. import sys
   
7. 
   
8. 
   
9. # SET THIS TO YOUR CASE NUMBER
   
10. my_case_num = 2090208000
    
11. # SET THIS TO THE RANGE YOU WANT TO DO THE SEARCH
    
12. search_range = 20
    
13. 
    
14. 
    
15. case_per_thread = 5
    
16. start = my_case_num - search_range
    
17. end = my_case_num + search_range
    
18. candidates = set(['Card Was Delivered To Me By The Post Office', 'New Card Is Being Produced', 'Card Was Mailed To Me',
    
19.                   'Card Was Picked Up By The United States Postal Service', 'Case Was Approved'])
    
20. other = set(['', ' ', 'Case Rejected Because I Sent An Incorrect Fee', 'Fees Were Waived',
    
21.              'Request for Initial Evidence Was Mailed', 'Fingerprint Fee Was Received'])
    
22. pending = 'Case Was Received'
    
23. url_prefix = '' // TODO
    .google  и
24. message = []
    
25. 
    
26. def getStatusMessage(thred_name, case_num):
    
27.     for num in range(case_num, case_num+case_per_thread)
    
28.         url = url_prefix + str(num)
    
29.         response = requests.get(url)
    
30.         bs = BeautifulSoup(response.text, 'html.parser')
    
31.         temp = bs.find(class_='col-lg-12 appointment-sec center')
    
32.         if not temp:
    
33.             print("error")
    
34.             exit()
    
35.         content = temp.find('p').get_text()
    
36.         status = temp.find('h1').get_text()
    
37.         message.append((num, status))
    
38. 
    
39. def printMessage(message):
    
40.     count_valid = 0
    
41.     count_other = 0
    
42.     count_pending = 0
    
43.     step = 0
    
44.     for msg in message:
    
45.         status = msg[1]
    
46.         if status in candidates:
    
47.             count_valid += 1
    
48.         elif status == pending:
    
49.             count_pending += 1
    
50.         else:
    
51.             count_other += 1
    
52.         step += 1
    
53.         print(msg[0], msg[1])
    
54.     print('total:', step)
    
55.     print('finished:', count_valid)
    
56.     print('pending:', count_pending)
    
57.     print('other:', count_other)
    
58.     print("Approval percentage: ", count_valid / step * 100)
    
59.     print("Pending percentage: ", count_pending / step * 100)
    
60.     print("Your case status: ", [tup for tup in message if tup[0] == my_case_num])
    
61. 
    
62. try:
    
63.     for case_num in range(start, end, case_per_thread):
    
64.         thread.start_new_thread(getStatusMessage, ("", case_num))
    
65. except:
    
66.    print "Error: unable to start thread"
    
67. 
    
68. while 1:
    
69.    if len(message) == search_range*2:
    
70.        print("\nFINISHED")
    
71.        message.sort(key=lambda tup: tup[0])
    
72.        printMessage(message)
    
73.        exit()
    
74.    else:
    
75.        i = int( (len(message) / (search_range*2)) * 100 )
    
76.        sys.stdout.write('\r')
    
77.        sys.stdout.write("[%-100s] %d%%" % ('=' * i, i))
    
78.        sys.stdout.flush()
    
79.        sleep(0.25)
    

复制代码