Skip to content

Instantly share code, notes, and snippets.

@elliotwesoff
Last active June 7, 2021 00:07
Show Gist options
  • Save elliotwesoff/4ef3e962f8e46b7078bc31ff47f5ce3c to your computer and use it in GitHub Desktop.
Save elliotwesoff/4ef3e962f8e46b7078bc31ff47f5ce3c to your computer and use it in GitHub Desktop.
Naruto manga downloader
import time
import pathlib
import requests
from optparse import OptionParser
base_url = 'https://official-complete-2.eorzea.us/manga/Naruto'
consecutive_error_threshold = 5
def setup():
parser = OptionParser()
parser.add_option('-s', '--chapter-start', dest='chapter_start', default=1)
parser.add_option('-e', '--chapter-end', dest='chapter_end', default=700)
parser.add_option('-o', '--output-path', dest='output_path', default='naruto')
parser.add_option('-c', '--chapter-wait-duration', dest='chapter_sleep_duration', default=30)
parser.add_option('-p', '--page-wait-duration', dest='page_sleep_duration', default=1)
return parser.parse_args()
def page_url(chapter, page):
# 0-pad the chapters and pages as necessary.
return f'{base_url}/{chapter:04}-{page:03}.png'
def main():
(options, args) = setup()
output_path = options.output_path
chapters = range(int(options.chapter_start), int(options.chapter_end) + 1)
chapter_sleep_duration = int(options.chapter_sleep_duration)
page_sleep_duration = int(options.page_sleep_duration)
consecutive_error_counter = 0
for chapter in chapters:
print(f'starting download for chapter {chapter}')
chapter_path = f'{output_path}/{chapter}'
pathlib.Path(chapter_path).mkdir(exist_ok=True, parents=True) # mkdir -p
page = 1
while True:
if consecutive_error_counter >= consecutive_error_threshold:
raise Exception(f'we got {consecutive_error_counter} errors in a row. something is wrong!')
print(f'downloading page {page}')
request = requests.get(page_url(chapter, page), stream=True)
if request.ok:
with open(f'{chapter_path}/{page}.png', 'wb') as f:
for chunk in request.iter_content(chunk_size=128):
f.write(chunk)
page += 1
consecutive_error_counter = 0
time.sleep(page_sleep_duration)
else:
print(f'error getting page #{page}! (it probably doesn\'t exist)')
print(f'received status code: {request.status_code} - {request.reason}. next chapter!')
consecutive_error_counter += 1
break
print(f'wait {chapter_sleep_duration}s')
time.sleep(chapter_sleep_duration)
return 0
if __name__ == '__main__':
try:
exit(main())
except KeyboardInterrupt:
print('abort!')
exit(1)
except Exception as e:
raise e
Copy link

ghost commented Jun 2, 2021

thank you!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment