中级农民
- 积分
- 290
- 大米
- 颗
- 鳄梨
- 个
- 水井
- 尺
- 蓝莓
- 颗
- 萝卜
- 根
- 小米
- 粒
- 学分
- 个
- 注册时间
- 2017-2-8
- 最后登录
- 1970-1-1
|
- #!/usr/bin/python
- """
- Crawl data from USCIS website
- https://egov.uscis.gov/casestatus/mycasestatus.do
- """
- import os
- import time
- import sqlite3
- import random.
- import urllib2
- from bs4 import BeautifulSoup, Comment
- from mechanize import Browser
- from dateutil.parser import parse
- import pandas as pd
- from datetime import datetime
- import numpy as np
- from pytz import timezone
- ..
- ADDRESS = 'https://egov.uscis.gov/casestatus/mycasestatus.do'
- default_ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
- class Crawler:
- def __init__(self,start,end, prefix, address = ADDRESS):
- self.url = ADDRESS
- self.browser = Browser()
- self.start = start
- self.end = end ..
- self.headers= ['receivd','approved',
- 'produce','mailed','rejected','amended',.--
- 'transferred','exist','update']. Χ
- #self.Init(start,end,prefix)
- #self.Update(prefix).--
- def __del__(self):. check 1point3acres for more.
- pass
- def getcontent(self, id):. 1point 3acres
- '''Get contents from html page for given case id.'''
- # randomize headers
- #try:. 1point3acres.com
- # header = {"Connection": "close", "User-Agent": ua.random}.google и
- #except:
- header = {"Connection": "close", "User-Agent": default_ua}
- # estabish connection.
- request = urllib2.Request(self.url, None, header)
- self.browser.open(request)
- self.browser.select_form(nr = 0)
- self.browser['appReceiptNum'] = id
- # get response for a case.--
- response = self.browser.submit()
- content = response.read()
- return content
- . .и
- def getstatus(self, text):
- if 'received your form i-765' in text:
- return 0
- elif 'approved' in text:
- return 1. 1point 3acres
- elif 'ordered your new card' in text:. 1point3acres
- return 2. Χ
- elif 'new card' in text:
- return 3
- elif 'rejected' in text:
- return 4. .и
- elif 'received your correspondence for form i-765' in text:. ----
- return 5
- elif 'transferred your form i-765' in text:
- return 6
- else:
- return 7
- def gettext(self, s):
- try:
- return parse(" ".join(s.split()[1:4])).date() ..
- except:. check 1point3acres for more.
- return None
- .
- def getinfo(self, id):. 1point 3acres
- content = self.getcontent(id)
- tag = BeautifulSoup(content,'lxml').find_all('p')[0]
- text = tag.get_text().replace(',', '').lower()
- .
- if id.lower() not in text: return id, None, None
- . From 1point 3acres bbs
- try:
- status = self.getstatus(text)
- date = self.gettext(text)
- except :
- status, date = None, None
- return id, status, date
- def Init(self,start,end,prefix):
- fname = 'book.csv'.google и
- headers = ['receivd','approved',
- 'produce','mail','rejected','amended',
- 'transferred','exist']
- if os.path.exists(fname):. Χ
- self.df = pd.read_csv('book.csv',names=headers,skiprows=[0])
- else:
- self.df = pd.DataFrame(columns = headers)
- #print self.df
- for idx in range(start,end):. 1point3acres.com
- if idx in self.df.index: continue
- id, status, date = self.getinfo(prefix+'{:06d}'.format(idx))
- if status is None:
- self.df.loc[idx] = [None]*(len(headers)-1)+[False]. Waral dи,
- else:.1point3acres
- d = [None]*(len(headers)-1) + [True]
- if status<(len(headers)-1): d[status] = date. From 1point 3acres bbs
- self.df.loc[idx] = d
- #print id,status,date
- if idx%20==0: self.df.to_csv('book.csv').--
- def Print(self,prefix,start,end):
- headers = self.headers.
- for idx in range(start,end+1):
- id, status, date = self.getinfo(prefix+'{:06d}'.format(idx))
- .1point3acres
- if 0<status<len(headers)-2:
- print prefix+'{:06d}'.format(idx),date,headers[status].--
- else:
- print prefix+'{:06d}'.format(idx),'not available'
- import os.path
- if __name__ == "__main__":
- crawler = Crawler(start=290000, end=307000, prefix='YSC1790'). 1point 3 acres
- crawler.Print('YSC1790',295800,295900)
|
|
狗仔卡
收藏
淘帖
好苗
杂草
提升卡
置顶卡
关贴卡
开贴卡
变色卡
千斤顶
楼主
