Merge 'rg3/master' into fork_master
This commit is contained in:
Filippo Valsorda
commit
3c6ffbaedb
12 changed files with 144 additions and 113 deletions
|
|
@ -334,8 +334,11 @@ class FileDownloader(object):
|
|||
template_dict['epoch'] = int(time.time())
|
||||
template_dict['autonumber'] = u'%05d' % self._num_downloads
|
||||
|
||||
template_dict = dict((key, u'NA' if val is None else val) for key, val in template_dict.items())
|
||||
template_dict = dict((k, sanitize_filename(compat_str(v), self.params.get('restrictfilenames'))) for k,v in template_dict.items())
|
||||
sanitize = lambda k,v: sanitize_filename(
|
||||
u'NA' if v is None else compat_str(v),
|
||||
restricted=self.params.get('restrictfilenames'),
|
||||
is_id=(k==u'id'))
|
||||
template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items())
|
||||
|
||||
filename = self.params['outtmpl'] % template_dict
|
||||
return filename
|
||||
|
|
|
|||
|
|
@ -1671,7 +1671,7 @@ class YahooSearchIE(InfoExtractor):
|
|||
class YoutubePlaylistIE(InfoExtractor):
|
||||
"""Information Extractor for YouTube playlists."""
|
||||
|
||||
_VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
|
||||
_VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]{10,})(?:/.*?/([0-9A-Za-z_-]+))?.*'
|
||||
_TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
|
||||
_VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&([^&"]+&)*list=.*?%s'
|
||||
_MORE_PAGES_INDICATOR = r'yt-uix-pager-next'
|
||||
|
|
@ -2803,13 +2803,13 @@ class SoundcloudIE(InfoExtractor):
|
|||
def __init__(self, downloader=None):
|
||||
InfoExtractor.__init__(self, downloader)
|
||||
|
||||
def report_webpage(self, video_id):
|
||||
def report_resolve(self, video_id):
|
||||
"""Report information extraction."""
|
||||
self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
|
||||
self._downloader.to_screen(u'[%s] %s: Resolving id' % (self.IE_NAME, video_id))
|
||||
|
||||
def report_extraction(self, video_id):
|
||||
"""Report information extraction."""
|
||||
self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
|
||||
self._downloader.to_screen(u'[%s] %s: Retrieving stream' % (self.IE_NAME, video_id))
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
|
|
@ -2818,65 +2818,47 @@ class SoundcloudIE(InfoExtractor):
|
|||
return
|
||||
|
||||
# extract uploader (which is in the url)
|
||||
uploader = mobj.group(1).decode('utf-8')
|
||||
uploader = mobj.group(1)
|
||||
# extract simple title (uploader + slug of song title)
|
||||
slug_title = mobj.group(2).decode('utf-8')
|
||||
slug_title = mobj.group(2)
|
||||
simple_title = uploader + u'-' + slug_title
|
||||
|
||||
self.report_webpage('%s/%s' % (uploader, slug_title))
|
||||
self.report_resolve('%s/%s' % (uploader, slug_title))
|
||||
|
||||
request = compat_urllib_request.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
|
||||
url = 'http://soundcloud.com/%s/%s' % (uploader, slug_title)
|
||||
resolv_url = 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
|
||||
request = compat_urllib_request.Request(resolv_url)
|
||||
try:
|
||||
webpage = compat_urllib_request.urlopen(request).read()
|
||||
info_json_bytes = compat_urllib_request.urlopen(request).read()
|
||||
info_json = info_json_bytes.decode('utf-8')
|
||||
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
||||
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
|
||||
return
|
||||
|
||||
info = json.loads(info_json)
|
||||
video_id = info['id']
|
||||
self.report_extraction('%s/%s' % (uploader, slug_title))
|
||||
|
||||
# extract uid and stream token that soundcloud hands out for access
|
||||
mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage)
|
||||
if mobj:
|
||||
video_id = mobj.group(1)
|
||||
stream_token = mobj.group(2)
|
||||
streams_url = 'https://api.sndcdn.com/i1/tracks/' + str(video_id) + '/streams?client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
|
||||
request = compat_urllib_request.Request(streams_url)
|
||||
try:
|
||||
stream_json_bytes = compat_urllib_request.urlopen(request).read()
|
||||
stream_json = stream_json_bytes.decode('utf-8')
|
||||
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
||||
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
|
||||
return
|
||||
|
||||
# extract unsimplified title
|
||||
mobj = re.search('"title":"(.*?)",', webpage)
|
||||
if mobj:
|
||||
title = mobj.group(1).decode('utf-8')
|
||||
else:
|
||||
title = simple_title
|
||||
|
||||
# construct media url (with uid/token)
|
||||
mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
|
||||
mediaURL = mediaURL % (video_id, stream_token)
|
||||
|
||||
# description
|
||||
description = u'No description available'
|
||||
mobj = re.search('track-description-value"><p>(.*?)</p>', webpage)
|
||||
if mobj:
|
||||
description = mobj.group(1)
|
||||
|
||||
# upload date
|
||||
upload_date = None
|
||||
mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage)
|
||||
if mobj:
|
||||
try:
|
||||
upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
|
||||
except Exception as err:
|
||||
self._downloader.to_stderr(compat_str(err))
|
||||
|
||||
# for soundcloud, a request to a cross domain is required for cookies
|
||||
request = compat_urllib_request.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
|
||||
streams = json.loads(stream_json)
|
||||
mediaURL = streams['http_mp3_128_url']
|
||||
|
||||
return [{
|
||||
'id': video_id.decode('utf-8'),
|
||||
'id': info['id'],
|
||||
'url': mediaURL,
|
||||
'uploader': uploader.decode('utf-8'),
|
||||
'upload_date': upload_date,
|
||||
'title': title,
|
||||
'uploader': info['user']['username'],
|
||||
'upload_date': info['created_at'],
|
||||
'title': info['title'],
|
||||
'ext': u'mp3',
|
||||
'description': description.decode('utf-8')
|
||||
'description': info['description'],
|
||||
}]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -305,7 +305,7 @@ def parseOpts():
|
|||
action='store_true', dest='autonumber',
|
||||
help='number downloaded files starting from 00000', default=False)
|
||||
filesystem.add_option('-o', '--output',
|
||||
dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout.')
|
||||
dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout. Can also be used to download to a different directory, for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .')
|
||||
filesystem.add_option('--restrict-filenames',
|
||||
action='store_true', dest='restrictfilenames',
|
||||
help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
|
||||
|
|
|
|||
|
|
@ -317,9 +317,10 @@ def timeconvert(timestr):
|
|||
timestamp = email.utils.mktime_tz(timetuple)
|
||||
return timestamp
|
||||
|
||||
def sanitize_filename(s, restricted=False):
|
||||
def sanitize_filename(s, restricted=False, is_id=False):
|
||||
"""Sanitizes a string so it could be used as part of a filename.
|
||||
If restricted is set, use a stricter subset of allowed characters.
|
||||
Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
|
||||
"""
|
||||
def replace_insane(char):
|
||||
if char == '?' or ord(char) < 32 or ord(char) == 127:
|
||||
|
|
@ -337,14 +338,15 @@ def sanitize_filename(s, restricted=False):
|
|||
return char
|
||||
|
||||
result = u''.join(map(replace_insane, s))
|
||||
while '__' in result:
|
||||
result = result.replace('__', '_')
|
||||
result = result.strip('_')
|
||||
# Common case of "Foreign band name - English song title"
|
||||
if restricted and result.startswith('-_'):
|
||||
result = result[2:]
|
||||
if not result:
|
||||
result = '_'
|
||||
if not is_id:
|
||||
while '__' in result:
|
||||
result = result.replace('__', '_')
|
||||
result = result.strip('_')
|
||||
# Common case of "Foreign band name - English song title"
|
||||
if restricted and result.startswith('-_'):
|
||||
result = result[2:]
|
||||
if not result:
|
||||
result = '_'
|
||||
return result
|
||||
|
||||
def orderedSet(iterable):
|
||||
|
|
@ -504,3 +506,6 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
|
|||
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
|
||||
resp.msg = old_resp.msg
|
||||
return resp
|
||||
|
||||
https_request = http_request
|
||||
https_response = http_response
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue