diff --git a/rov/__init__.py b/rov/__init__.py index da94f84..d10c58d 100644 --- a/rov/__init__.py +++ b/rov/__init__.py @@ -12,6 +12,8 @@ import shutil import sys import csv +import lzma +from io import BytesIO import urllib import urllib.request as request @@ -64,11 +66,12 @@ 'https://rpki.gin.ntt.net/api/export.json' ] RPKI_ARCHIVE_URLS = [ - 'https://ftp.ripe.net/ripe/rpki/afrinic.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv', - 'https://ftp.ripe.net/ripe/rpki/apnic.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv', - 'https://ftp.ripe.net/ripe/rpki/arin.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv', - 'https://ftp.ripe.net/ripe/rpki/lacnic.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv', - 'https://ftp.ripe.net/ripe/rpki/ripencc.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv', + 'https://ftp.ripe.net/rpki/afrinic.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv.xz', + 'https://ftp.ripe.net/rpki/apnic.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv.xz', + 'https://ftp.ripe.net/rpki/arin.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv.xz', + 'https://ftp.ripe.net/rpki/lacnic.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv.xz', + 'https://ftp.ripe.net/rpki/ripencc.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv.xz', + ] DEFAULT_DELEGATED_URLS = [ 'https://www.nro.net/wp-content/uploads/delegated-stats/nro-extended-stats' @@ -459,17 +462,25 @@ def download_databases(self, overwrite=True): # all files from RIPE's RPKI archive have the same name # 'roas.csv', change it with the tal name - if fname == 'roas.csv': - fname = guess_ta_name(url)+'.csv' + if fname == 'roas.csv.xz': + fname = guess_ta_name(url)+".csv" if os.path.exists(folder+fname) and not overwrite: continue sys.stderr.write(f'Downloading: {url}\n') + try: - with closing(request.urlopen(url)) as r: - with open(folder+fname, 'wb') as f: - shutil.copyfileobj(r, f) + # to separete csv.xz file to decompress + if "roas.csv.xz" in url: + with closing(request.urlopen(url)) as response: + with lzma.open(BytesIO(response.read())) as r: + with open(folder+fname, 'wb') as f: + shutil.copyfileobj(r,f) + else: + with closing(request.urlopen(url)) as r: + with open(folder+fname, 'wb') as f: + shutil.copyfileobj(r, f) except urllib.error.URLError: sys.stderr.write(f'Error {url} is not available.\n') - + diff --git a/setup.py b/setup.py index 7b276e2..e1d15da 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ readme = f.read() setup( - version = '0.4.0', + version = '0.5.0', name = 'rov', author = 'Romain Fontugne', author_email = 'romain.fontugne@gmail.com', @@ -17,7 +17,7 @@ install_requires=[ 'appdirs', 'py-radix', - 'portion', + 'portion' ], entry_points={'console_scripts': ['rov = rov.__main__:main']},