Create surfer.py
This commit is contained in:
Lico San
@@ -0,0 +1,229 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import requests, re, copy
|
||||
from bs4 import BeautifulSoup
|
||||
from requests.auth import HTTPBasicAuth
|
||||
from time import sleep
|
||||
|
||||
class surfer:
|
||||
|
||||
def __init__(self, base_url='', cookies=True, allow_redirects=False, credentials=None, debug=False):
|
||||
self.base_url = base_url
|
||||
self.debug = debug
|
||||
self.cookies = cookies
|
||||
self.allow_redirects = allow_redirects
|
||||
self.credentials = credentials
|
||||
|
||||
|
||||
def surf(self, waves, entry_wave):
|
||||
self.cookiejar = None
|
||||
self.extracted_values = {}
|
||||
self.waves = waves
|
||||
self.next_wave = entry_wave
|
||||
while self.next_wave is not None :
|
||||
if ('delay' in self.waves[self.next_wave]) and (self.waves[self.next_wave]['delay']):
|
||||
sleep(self.waves[self.next_wave]['delay'])
|
||||
self._surf_wave()
|
||||
|
||||
|
||||
def _surf_wave(self):
|
||||
self.cur_wave = self.next_wave
|
||||
if self.debug: print('BASE=%s | WAVEURL=%s' %(self.base_url, self.waves[self.cur_wave]['sub_url']))
|
||||
if self.waves[self.cur_wave]['sub_url'].lower().startswith('http'):
|
||||
url = self.waves[self.cur_wave]['sub_url']
|
||||
else:
|
||||
url = self.base_url+self.waves[self.cur_wave]['sub_url']
|
||||
self._prepare_params()
|
||||
self._prepare_headers()
|
||||
req_meth_name = self.waves[self.cur_wave]['method'].lower()
|
||||
req_meth = requests.__dict__[req_meth_name]
|
||||
|
||||
if self.debug: print('Surfing to %s\n Method:%s\n Headers:%s\n Params:%s\n Cookies:%s\n' %(url, req_meth_name, self.headers, self.params, self.cookiejar))
|
||||
if self.credentials:
|
||||
r = req_meth( url,
|
||||
allow_redirects = self.allow_redirects,
|
||||
headers = self.headers,
|
||||
cookies=self.cookiejar,
|
||||
params = self.params,
|
||||
auth=(self.credentials['user'], self.credentials['pass'])
|
||||
)
|
||||
else:
|
||||
r = req_meth( url,
|
||||
allow_redirects = self.allow_redirects,
|
||||
headers = self.headers,
|
||||
cookies=self.cookiejar,
|
||||
params = self.params
|
||||
)
|
||||
|
||||
if r.cookies:
|
||||
self.cookiejar = r.cookies
|
||||
if self.debug: print('Cookies after wave %s : %s' %(self.cur_wave, self.cookiejar))
|
||||
|
||||
self.next_wave = self.waves[self.cur_wave]['next_wave']
|
||||
if r.status_code != 200:
|
||||
if self.debug: print('Error; http status=%s' %r.status_code)
|
||||
if callable(self.waves[self.cur_wave]['if_not_200']):
|
||||
self.waves[self.cur_wave]['if_not_200'](r, self)
|
||||
else:
|
||||
if 'text' == self.waves[self.cur_wave]['parsing']:
|
||||
if self.debug: print('Parsing text...')
|
||||
data = r.text
|
||||
self.extracted_values = {}
|
||||
self._regexp_extract(data)
|
||||
elif 'json' == self.waves[self.cur_wave]['parsing']:
|
||||
if self.debug: print('Parsing json...')
|
||||
data = r.json()
|
||||
# ?? extracting in json has no sense to me !?
|
||||
elif 'html' == self.waves[self.cur_wave]['parsing']:
|
||||
self.extracted_values = {}
|
||||
self._regexp_extract(r.text)
|
||||
data = BeautifulSoup(r.text, features="html.parser")
|
||||
self._html_xtract(data)
|
||||
elif 'xml' == self.waves[self.cur_wave]['parsing']:
|
||||
self.extracted_values = {}
|
||||
self._regexp_extract(r.text)
|
||||
data = BeautifulSoup(r.text, 'xml') #features="xml.parser"
|
||||
self._xml_xtract(data)
|
||||
|
||||
|
||||
if callable(self.waves[self.cur_wave]['test_response']):
|
||||
test_ok = self.waves[self.cur_wave]['test_response'](data, self)
|
||||
else:
|
||||
test_ok = True
|
||||
|
||||
if self.debug: print('Tested content:%s' %(('ok' if test_ok else 'not ok!')))
|
||||
|
||||
if test_ok and callable(self.waves[self.cur_wave]['if_test_ok']):
|
||||
self.waves[self.cur_wave]['if_test_ok'](data, self)
|
||||
elif (not test_ok) and callable(self.waves[self.cur_wave]['if_test_fail']):
|
||||
self.waves[self.cur_wave]['if_test_fail'](data, self)
|
||||
|
||||
|
||||
def _html_xtract(self, parsed_html):
|
||||
if 'extract' not in self.waves[self.cur_wave]: return
|
||||
for needle in self.waves[self.cur_wave]['extract']:
|
||||
if 'bs4_selectors' not in needle: continue
|
||||
node = parsed_html
|
||||
for selector in needle['bs4_selectors']:
|
||||
res = node.select(selector)
|
||||
if res:
|
||||
#if self.debug: print('Selector %s match ! %s' %(selector, res))
|
||||
node = res[0]
|
||||
if self.debug: print('Selector %s found !' %(selector))
|
||||
else:
|
||||
if self.debug: print('Selector %s NOT match !' %selector)
|
||||
break
|
||||
|
||||
if res:
|
||||
if ('bs4_getall' in needle) and needle['bs4_getall']:
|
||||
nodes = res
|
||||
extracts = []
|
||||
for node in nodes:
|
||||
if 'bs4_extractor' in needle and callable(needle['bs4_extractor']):
|
||||
extracts.append(needle['bs4_extractor'](node))
|
||||
else:
|
||||
extracts.append(node.string)
|
||||
elif 'bs4_getone' in needle:
|
||||
node = res[needle['bs4_getone']]
|
||||
if 'bs4_extractor' in needle and callable(needle['bs4_extractor']):
|
||||
extracts = needle['bs4_extractor'](node)
|
||||
else:
|
||||
extracts = node.string
|
||||
else:
|
||||
node = res[0]
|
||||
if 'bs4_extractor' in needle and callable(needle['bs4_extractor']):
|
||||
extracts = needle['bs4_extractor'](node)
|
||||
else:
|
||||
extracts = node.string
|
||||
|
||||
|
||||
if self.debug: print('HTML Needle Found : %s' %extracts)
|
||||
self.extracted_values[needle['name']] = extracts
|
||||
else:
|
||||
if self.debug: print('HTML Needle %s NOT Found!' %needle['name'])
|
||||
if needle['name'] not in self.extracted_values: self.extracted_values[needle['name']] = None
|
||||
if self.debug: print('HTML Extracted: %s' %self.extracted_values)
|
||||
|
||||
|
||||
def _xml_xtract(self, parsed_xml):
|
||||
if 'extract' not in self.waves[self.cur_wave]: return
|
||||
for needle in self.waves[self.cur_wave]['extract']:
|
||||
if 'bs4_selectors' not in needle: continue
|
||||
node = parsed_xml
|
||||
for selector in needle['bs4_selectors']:
|
||||
res = node.select(selector)
|
||||
if res:
|
||||
#if self.debug: print('Selector %s match ! %s' %(selector, res))
|
||||
node = res[0]
|
||||
if self.debug: print('Selector %s found !' %(selector))
|
||||
else:
|
||||
if self.debug: print('Selector %s NOT match !' %selector)
|
||||
break
|
||||
|
||||
if res:
|
||||
if ('bs4_getall' in needle) and needle['bs4_getall']:
|
||||
nodes = res
|
||||
extracts = []
|
||||
for node in nodes:
|
||||
if 'bs4_extractor' in needle and callable(needle['bs4_extractor']):
|
||||
extracts.append(needle['bs4_extractor'](node))
|
||||
else:
|
||||
extracts.append(node.string)
|
||||
elif 'bs4_getone' in needle:
|
||||
node = res[needle['bs4_getone']]
|
||||
if 'bs4_extractor' in needle and callable(needle['bs4_extractor']):
|
||||
extracts = needle['bs4_extractor'](node)
|
||||
else:
|
||||
extracts = node.string
|
||||
else:
|
||||
node = res[0]
|
||||
if 'bs4_extractor' in needle and callable(needle['bs4_extractor']):
|
||||
extracts = needle['bs4_extractor'](node)
|
||||
else:
|
||||
extracts = node.string
|
||||
|
||||
|
||||
if self.debug: print('XML Needle Found : %s' %extracts)
|
||||
self.extracted_values[needle['name']] = extracts
|
||||
else:
|
||||
if self.debug: print('XML Needle %s NOT Found!' %needle['name'])
|
||||
if needle['name'] not in self.extracted_values: self.extracted_values[needle['name']] = None
|
||||
if self.debug: print('XML Extracted: %s' %self.extracted_values)
|
||||
|
||||
|
||||
|
||||
def _regexp_extract(self, content):
|
||||
if 'extract' not in self.waves[self.cur_wave]: return
|
||||
for needle in self.waves[self.cur_wave]['extract']:
|
||||
if 'regexp' not in needle: continue
|
||||
if self.debug: print('Searching for: %s' %needle)
|
||||
matches = re.findall(needle['regexp'], content, needle['regexp_flags'])
|
||||
if matches:
|
||||
if self.debug: print('TEXT Needle Found : %s' %matches)
|
||||
if isinstance(matches[0], tuple):
|
||||
self.extracted_values[needle['name']] = matches[0][needle['regexp_group_index']]
|
||||
else:
|
||||
self.extracted_values[needle['name']] = matches[0]
|
||||
else:
|
||||
if self.debug: print('TEXT Needle %s NOT found!' %needle['name'])
|
||||
if needle['name'] not in self.extracted_values: self.extracted_values[needle['name']] = None
|
||||
if self.debug: print('TEXT Extracted: %s' %self.extracted_values)
|
||||
|
||||
|
||||
def _prepare_params(self):
|
||||
self.params = copy.deepcopy(self.waves[self.cur_wave]['params'])
|
||||
for k, v in self.extracted_values.items():
|
||||
for par,parval in self.params.items():
|
||||
if isinstance(v, list) or isinstance(v, tuple): v = ' '.join(v)
|
||||
self.params[par]=parval.replace('%%{%s}'%k.encode('utf-8', 'ignore'), v.encode('utf-8', 'ignore'))
|
||||
# If replacement in keys is necessary do it here
|
||||
|
||||
|
||||
def _prepare_headers(self):
|
||||
self.headers = copy.deepcopy(self.waves[self.cur_wave]['headers'])
|
||||
for k, v in self.extracted_values.items():
|
||||
if v == None: v=''
|
||||
for par,parval in self.headers.items():
|
||||
if isinstance(v, list) or isinstance(v, tuple): v = ' '.join(v)
|
||||
self.headers[par]=parval.replace('%%{%s}'%k.encode('utf-8', 'ignore'), v.encode('utf-8', 'ignore'))
|
||||
# If replacement in keys is necessary do it here
|
||||
|
||||
Reference in New Issue
Block a user