中级农民
- 积分
- 291
- 大米
- 颗
- 鳄梨
- 个
- 水井
- 尺
- 蓝莓
- 颗
- 萝卜
- 根
- 小米
- 粒
- 学分
- 个
- 注册时间
- 2017-2-8
- 最后登录
- 1970-1-1
|
- #!/usr/bin/python
- """
- Crawl data from USCIS website
- https://egov.uscis.gov/casestatus/mycasestatus.do
- """
- .--
- import os.--
- import time
- import sqlite3
- import random
- import urllib2
- from bs4 import BeautifulSoup, Comment
- from mechanize import Browser.1point3acres
- from dateutil.parser import parse
- import pandas as pd
- from datetime import datetime
- import numpy as np
- from pytz import timezone
- ADDRESS = 'https://egov.uscis.gov/casestatus/mycasestatus.do'
- default_ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
- class Crawler:
- def __init__(self,start,end, prefix, address = ADDRESS):
- self.url = ADDRESS.
- self.browser = Browser()
- self.start = start
- self.end = end
- self.headers= ['receivd','approved',
- 'produce','mailed','rejected','amended',
- 'transferred','exist','update']
- #self.Init(start,end,prefix).1point3acres
- #self.Update(prefix)
- def __del__(self):
- pass
- def getcontent(self, id):
- '''Get contents from html page for given case id.'''
- # randomize headers
- #try:
- # header = {"Connection": "close", "User-Agent": ua.random}
- #except:
- header = {"Connection": "close", "User-Agent": default_ua}
- # estabish connection
- request = urllib2.Request(self.url, None, header)
- self.browser.open(request)
- self.browser.select_form(nr = 0)
- self.browser['appReceiptNum'] = id
- # get response for a case
- response = self.browser.submit()
- content = response.read()
- return content
- def getstatus(self, text):
- if 'received your form i-765' in text:
- return 0
- elif 'approved' in text:.--
- return 1
- elif 'ordered your new card' in text:
- return 2
- elif 'new card' in text:
- return 3
- elif 'rejected' in text:
- return 4
- elif 'received your correspondence for form i-765' in text:
.1point3acres - return 5
- elif 'transferred your form i-765' in text:
- return 6
- else:
- return 7
- .
- def gettext(self, s):
- .
- try:
- return parse(" ".join(s.split()[1:4])).date()
- except:. 1point3acres.com
- return None. 1point 3acres
- def getinfo(self, id):
- content = self.getcontent(id)
- tag = BeautifulSoup(content,'lxml').find_all('p')[0]
- text = tag.get_text().replace(',', '').lower(). 1point 3 acres
- if id.lower() not in text: return id, None, None. From 1point 3acres bbs
- . check 1point3acres for more.
- try:
- status = self.getstatus(text)
- date = self.gettext(text)
- except :
- status, date = None, None. .и
- return id, status, date
- . check 1point3acres for more.
- def Init(self,start,end,prefix):
- fname = 'book.csv'
- headers = ['receivd','approved',
- 'produce','mail','rejected','amended',
- 'transferred','exist']
- if os.path.exists(fname):
- self.df = pd.read_csv('book.csv',names=headers,skiprows=[0]). Waral dи,
- else:
- self.df = pd.DataFrame(columns = headers).1point3acres
- #print self.df-baidu 1point3acres
- for idx in range(start,end):
- if idx in self.df.index: continue
- id, status, date = self.getinfo(prefix+'{:06d}'.format(idx))
- if status is None:. 1point3acres.com
- self.df.loc[idx] = [None]*(len(headers)-1)+[False]. From 1point 3acres bbs
- else:
- d = [None]*(len(headers)-1) + [True]
- if status<(len(headers)-1): d[status] = date
- self.df.loc[idx] = d. ----
- #print id,status,date
- if idx%20==0: self.df.to_csv('book.csv'). check 1point3acres for more.
- def Print(self,prefix,start,end):
- headers = self.headers
- for idx in range(start,end+1):
- id, status, date = self.getinfo(prefix+'{:06d}'.format(idx))
. 1point3acres.com - if 0<status<len(headers)-2:
- print prefix+'{:06d}'.format(idx),date,headers[status]
- else:.1point3acres
- print prefix+'{:06d}'.format(idx),'not available'
- . 1point 3acres
- . 1point3acres
- import os.path
- if __name__ == "__main__":
- crawler = Crawler(start=290000, end=307000, prefix='YSC1790')
- crawler.Print('YSC1790',295800,295900)
|
|
狗仔卡
收藏
淘帖
好苗
杂草
提升卡
置顶卡
关贴卡
开贴卡
变色卡
千斤顶
楼主
