#!/usr/bin/env python """gitlab-to-codecommit: script to migrate repositories from GitLab to AWS CodeCommit""" import requests import argparse import logging import os import sys import urllib.parse from gitlab_codecommit_migration import gitlab_codecommit_migration parser = argparse.ArgumentParser() parser.add_argument('--gitlab-access-token', required=True) parser.add_argument('--gitlab-url', required=True, help='url starting with http:// or https://') parser.add_argument('--chime-webhook-url', required=False, default="", help='send repo notifications to Chime room through this webhook.') parser.add_argument('--local-temp', required=False, default='/tmp', help='local path on filesystem to store to store repositories temporarily. Default is /tmp') parser.add_argument('--verbose', required=False, default=False, help='set logging to debug for verbose output') group_projects = parser.add_mutually_exclusive_group(required=True) group_projects.add_argument('--repository-names', nargs='+', help='names fo repositories in the form of "namespace/repository-name" separated by space.') group_projects.add_argument('--all', help='mirror all repositories', action='store_true') group_projects.add_argument('--users', nargs='+', help='mirror all repositories for the given users (separated by space)') group_projects.add_argument('--groups', nargs='+', help='mirror all repositories for the groups (separate by space)') args = parser.parse_args() logger = logging.getLogger(__name__) log_level = logging.INFO if args.verbose: log_level = logging.DEBUG logging.basicConfig(stream=sys.stdout, level=log_level) gitlab_url = args.gitlab_url if 'http' not in gitlab_url: print('gitlab url should start with http:// or https://') sys.exit(1) gitlab_access_token = args.gitlab_access_token gitlab_api = f'{gitlab_url}/api/v4' gitlab_project_url = f'{gitlab_api}/projects?per_page=100' gitlab_project_by_name_url = f'{gitlab_api}/projects' gitlab_project_by_groups = f'{gitlab_api}/groups' gitlab_project_by_users = f'{gitlab_api}/users' chime_webhook_url = args.chime_webhook_url path_local_target_clone_repos = args.local_temp projects_all = args.all projects_users = args.users projects_groups = args.groups python_script_dir = os.path.dirname(os.path.realpath(__file__)) headers = { 'Private-Token': gitlab_access_token, 'cache-control': "no-cache" } projects = [] if projects_all: logger.info(f'get_projects_url: {gitlab_project_url}') r = requests.request("GET", gitlab_project_url, data='', headers=headers) projects.extend(r.json()) # use GitLab API links to get all results in case of paging while r.links and r.links.get('next') and r.links.get('next').get('url'): r = requests.request("GET", r.links.get('next').get('url'), data='', headers=headers) projects.extend(r.json()) logger.info(f'found {len(projects)} projects in GitLab') elif projects_users: for user in projects_users: r = requests.request("GET", f'{gitlab_project_by_users}/{user}/projects', data='', headers=headers) projects.extend(r.json()) while r.links and r.links.get('next') and r.links.get('next').get('url'): r = requests.request("GET", r.links.get('next').get('url'), data='', headers=headers) projects.extend(r.json()) logger.info(f'found {len(projects)} projects in GitLab for user: {user}') if len(projects_users) > 1: logger.info(f'found total number of {len(projects)} projects in GitLab for users: {projects_users}') elif projects_groups: for group in projects_groups: r = requests.request("GET", f'{gitlab_project_by_groups}/{group}/projects', data='', headers=headers) projects.extend(r.json()) while r.links and r.links.get('next') and r.links.get('next').get('url'): r = requests.request("GET", r.links.get('next').get('url'), data='', headers=headers) projects.extend(r.json()) logger.info(f'found {len(projects)} projects in GitLab for group: {group}') if len(projects_groups) > 1: logger.info(f'found total number of {len(projects)} projects in GitLab for groups: {projects_groups}') else: for path_with_namespace in args.repository_names: urlencode_project = f'{gitlab_project_by_name_url}/{urllib.parse.quote(path_with_namespace, safe="")}' logger.info(f'project: {urlencode_project}') r = requests.request("GET", urlencode_project, data='', headers=headers) projects.append(r.json()) gitlab_codecommit_migration.migrate(projects, path_local_target_clone_repos=path_local_target_clone_repos, chime_webhook_url=chime_webhook_url, log_level=log_level)