Skip to content

Add url_post template parameter for remote cdx api #587

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Add url_post template parameter for remote cdx api #587

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
11dba03
Add url_post template parameter for remote cdx api
kaij Oct 30, 2020
853eedc
Fix url_post url encoding
kaij Dec 31, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion pywb/warcserver/index/indexsource.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from six.moves.urllib.parse import quote_plus
from six.moves.urllib.parse import quote_plus, quote, parse_qs, urlparse
from warcio.timeutils import PAD_14_DOWN, http_date_to_timestamp, pad_timestamp, timestamp_now, timestamp_to_http_date

from pywb.utils.binsearch import iter_range
Expand Down Expand Up @@ -112,6 +112,8 @@ def init_from_config(cls, config):
#=============================================================================
class RemoteIndexSource(BaseIndexSource):
CDX_MATCH_RX = re.compile('^cdxj?\+(?P<url>https?\:.*)')
POSTDATA_MATCH_RX = re.compile('.*?[?&](?P<post_key>__wb_post_data|__warc_post_data|__wb_json_data)'\
'=(?P<post_data>[^&]+).*$')

def __init__(self, api_url, replay_url, url_field='load_url', closest_limit=10):
self.api_url = api_url
Expand All @@ -121,15 +123,33 @@ def __init__(self, api_url, replay_url, url_field='load_url', closest_limit=10):
self._init_sesh()

def _get_api_url(self, params):
self.add_url_post_param(params)
api_url = res_template(self.api_url, params)

if 'closest' in params and self.closest_limit:
api_url += '&limit=' + str(self.closest_limit)

if 'matchType' in params:
api_url += '&matchType=' + params.get('matchType')

self.logger.info(api_url)
return api_url

def add_url_post_param(self, params):
# extract POST data value from urlkey and compose url_post parameter
key_str = params['key'].decode('utf-8')
match_post = re.match(self.POSTDATA_MATCH_RX, key_str)
params['url_post'] = quote(params['url'])

if match_post and match_post.groupdict() is not None:
url_query = parse_qs(urlparse(params['url']).query)
post_key = match_post.groupdict()['post_key']
post_data = match_post.groupdict()['post_data']
if len(url_query.keys()) == 0:
params['url_post'] += quote('?%s=%s' % (post_key, post_data))
else:
params['url_post'] += quote('&%s=%s' % (post_key, post_data))

def load_index(self, params):
api_url = self._get_api_url(params)
try:
Expand Down