中级农民
- 积分
- 291
- 大米
- 颗
- 鳄梨
- 个
- 水井
- 尺
- 蓝莓
- 颗
- 萝卜
- 根
- 小米
- 粒
- 学分
- 个
- 注册时间
- 2017-2-8
- 最后登录
- 1970-1-1
|
- #!/usr/bin/python. Χ
- """. 1point3acres
- Crawl data from USCIS website. check 1point3acres for more.
- https://egov.uscis.gov/casestatus/mycasestatus.do
- """
- import os
- import time.
- import sqlite3. 1point 3 acres
- import random
- import urllib2
- from bs4 import BeautifulSoup, Comment
- from mechanize import Browser
- from dateutil.parser import parse
- import pandas as pd
- from datetime import datetime
- import numpy as np
- from pytz import timezone
- ADDRESS = 'https://egov.uscis.gov/casestatus/mycasestatus.do'
.-- - default_ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
- class Crawler:
- def __init__(self,start,end, prefix, address = ADDRESS):. 1point3acres
- self.url = ADDRESS
- self.browser = Browser()
- self.start = start
- self.end = end
- self.headers= ['receivd','approved',
-baidu 1point3acres - 'produce','mailed','rejected','amended',. 1point 3acres
- 'transferred','exist','update']
- #self.Init(start,end,prefix)
- #self.Update(prefix). check 1point3acres for more.
- def __del__(self):
- pass
- . ----
- def getcontent(self, id):
- '''Get contents from html page for given case id.'''
- # randomize headers
- ..
- #try:
- # header = {"Connection": "close", "User-Agent": ua.random} ..
- #except:
- header = {"Connection": "close", "User-Agent": default_ua}
- # estabish connection
- request = urllib2.Request(self.url, None, header)
- self.browser.open(request).google и
- self.browser.select_form(nr = 0).--
- self.browser['appReceiptNum'] = id
- # get response for a case
- response = self.browser.submit().--
- content = response.read()
- return content
- def getstatus(self, text):
- if 'received your form i-765' in text:
- return 0
- elif 'approved' in text:
- return 1.--
- elif 'ordered your new card' in text:
- return 2
- elif 'new card' in text:
- return 3
- elif 'rejected' in text:
- return 4
- elif 'received your correspondence for form i-765' in text:. 1point 3acres
- return 5. .и
- elif 'transferred your form i-765' in text:
- return 6
- else:
- return 7. 1point 3acres
- ..
- def gettext(self, s):. Χ
- try:
- return parse(" ".join(s.split()[1:4])).date()
- except:
- return None
- ..
- def getinfo(self, id):
- content = self.getcontent(id)
- . 1point3acres.com
- tag = BeautifulSoup(content,'lxml').find_all('p')[0]
- text = tag.get_text().replace(',', '').lower()
- if id.lower() not in text: return id, None, None.--
- try:
- status = self.getstatus(text)
- date = self.gettext(text)
- except :. 1point3acres
- status, date = None, None
- return id, status, date. .и
- def Init(self,start,end,prefix):.1point3acres
- fname = 'book.csv'. 1point 3 acres
- headers = ['receivd','approved',
- 'produce','mail','rejected','amended',
- 'transferred','exist']
- if os.path.exists(fname):. 1point3acres.com
- self.df = pd.read_csv('book.csv',names=headers,skiprows=[0])
- else:. 1point3acres.com
- self.df = pd.DataFrame(columns = headers)
- #print self.df
- for idx in range(start,end):-baidu 1point3acres
- if idx in self.df.index: continue.--
- id, status, date = self.getinfo(prefix+'{:06d}'.format(idx))
- if status is None:
- self.df.loc[idx] = [None]*(len(headers)-1)+[False]
- else:. Χ
- d = [None]*(len(headers)-1) + [True]
- if status<(len(headers)-1): d[status] = date. .и
- self.df.loc[idx] = d-baidu 1point3acres
- #print id,status,date
- if idx%20==0: self.df.to_csv('book.csv')
- def Print(self,prefix,start,end):
- . From 1point 3acres bbs
- headers = self.headers
- for idx in range(start,end+1):
- id, status, date = self.getinfo(prefix+'{:06d}'.format(idx))
- if 0<status<len(headers)-2:
- print prefix+'{:06d}'.format(idx),date,headers[status]
- else:
- print prefix+'{:06d}'.format(idx),'not available'
- . 1point 3 acres
- import os.path
- if __name__ == "__main__":
- crawler = Crawler(start=290000, end=307000, prefix='YSC1790'). Χ
- crawler.Print('YSC1790',295800,295900)
复制代码 给你发段我的code |
|