README.md

Get github repositories for an organisation

requirement

• python

install

• pip install requests

run

python fetch.py --organisation ministryofjustice --token ${token}

fetch.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
get the repositories of an organisation
"""
import os
import re
import json
import urllib.parse
import argparse

import requests


PWD = os.path.abspath(os.path.dirname(__file__))
BASE_URL = 'https://api.github.com'
REGEX = (
    '<(?P<next_url>.*)>; rel="next", '
    '<(?P<last_url>.*?page=(?P<total>\d+))>; rel="last"'
)


def get_total(token, organisation):
    rsp = requests.get(
        urllib.parse.urljoin(BASE_URL, 'orgs/{}/repos'.format(organisation)),
        headers={'Authorization': 'token {}'.format(token)})
    link = rsp.headers['Link']
    total = re.match(REGEX, link).groupdict()['total']
    return int(total)


def get_repos(token, organisation):
    repos = []
    for page in range(1, get_total(token, organisation) + 1):
        rsp = requests.get(
            urllib.parse.urljoin(
                BASE_URL,
                'orgs/ministryofjustice/repos?page={}'.format(page)
            ),
            headers={'Authorization': 'token {}'.format(token)}
        )
        repos += rsp.json()
    return repos


def argument_parser():
    parser = argparse.ArgumentParser(
        description='dfetch repo data from github')
    parser.add_argument('--token', '-t', help='token', required=True)
    parser.add_argument(
        '--organisation', '-o', help='organisation', required=True)
    return parser


def main(argv=None):
    args = argument_parser().parse_args(argv)
    print('fetch repo data from github')
    repos = get_repos(args.token, args.organisation)
    filename = os.path.join(PWD, 'repos.json')
    print('{} repositories found'.format(len(repos)))
    print('data saved to {}'.format(filename))
    with open(filename, 'w') as fp:
        json.dump(repos, fp, indent=2)


if __name__ == '__main__':
    main()