better logging and file naming
#!/usr/bin/env python3
"""
-Uses youtube-dl to download any videos in the tweets. It writes a tab separated mapping of
-URLs to filenames.
+This utility uses youtube-dl to download any videos in the tweets. Despite its name
+youtube-dl is able to download video from a wide variety of websites, including
+from Twitter itself. In addition to downloading the video a JSON metadata for
+each video is saved as well as a WebVTT transcript if available. It writes a tab separated
+mapping of URLs to filenames so that the URLs in tweets can be matched up again
+with the files on disk.
"""
import sys
@@ -27,19 +31,26 @@ ydl_opts = {
"writeinfojson": True,
"writesubtitles": True,
"writeautomaticsub": True,
- "outtmpl": "youtubedl/%(extractor)s/%(title)s.%(ext)s",
+ "outtmpl": "youtubedl/%(extractor)s/%(id)s/%(title)s.%(ext)s",
"download_archive": "youtubedl/archive.txt"
}
ydl = youtube_dl.YoutubeDL(ydl_opts)
results = open('youtubedl/results.tsv', 'w')
+seen = set()
for line in sys.stdin:
tweet = json.loads(line)
+ logger.info('analyzing %s', tweet['id_str'])
for e in tweet['entities']['urls']:
url = e.get('unshortened_url') or e['expanded_url']
+ if url in seen:
+ logging.info('already processed %s', url)
+ continue
+ logging.info('processing %s', url)
info = ydl.extract_info(url, download=True)
if info:
filename = ydl.prepare_filename(info)
ydl.download([url])
results.write("{}\t{}\n".format(url, filename))
+ seen.add(url)
