Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

RIPE RPKI archive format changed (.xz) #14 #17

Merged
merged 7 commits into from
Jan 26, 2024
Merged

RIPE RPKI archive format changed (.xz) #14 #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
093ea74
updated the urls and uncompress the files while reading it
MAVRICK-1 Jan 12, 2024
990b0e3
added lzma in install_requires
MAVRICK-1 Jan 12, 2024
692d4c7
added lzma
MAVRICK-1 Jan 12, 2024
82ba4ca
upated RKPI_ARCHIVE_URL
MAVRICK-1 Jan 12, 2024
29e115d
Update setup.py
MAVRICK-1 Jan 25, 2024
8b8af4f
Update requirements.txt
MAVRICK-1 Jan 25, 2024
4ce2416
bump version updated
MAVRICK-1 Jan 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 22 additions & 11 deletions rov/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
import shutil
import sys
import csv
import lzma
from io import BytesIO

import urllib
import urllib.request as request
Expand Down Expand Up @@ -64,11 +66,12 @@
'https://rpki.gin.ntt.net/api/export.json'
]
RPKI_ARCHIVE_URLS = [
'https://ftp.ripe.net/ripe/rpki/afrinic.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv',
'https://ftp.ripe.net/ripe/rpki/apnic.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv',
'https://ftp.ripe.net/ripe/rpki/arin.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv',
'https://ftp.ripe.net/ripe/rpki/lacnic.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv',
'https://ftp.ripe.net/ripe/rpki/ripencc.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv',
'https://ftp.ripe.net/rpki/afrinic.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv.xz',
'https://ftp.ripe.net/rpki/apnic.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv.xz',
'https://ftp.ripe.net/rpki/arin.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv.xz',
'https://ftp.ripe.net/rpki/lacnic.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv.xz',
'https://ftp.ripe.net/rpki/ripencc.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv.xz',

]
DEFAULT_DELEGATED_URLS = [
'https://www.nro.net/wp-content/uploads/delegated-stats/nro-extended-stats'
Expand Down Expand Up @@ -459,17 +462,25 @@ def download_databases(self, overwrite=True):

# all files from RIPE's RPKI archive have the same name
# 'roas.csv', change it with the tal name
if fname == 'roas.csv':
fname = guess_ta_name(url)+'.csv'
if fname == 'roas.csv.xz':
fname = guess_ta_name(url)+".csv"

if os.path.exists(folder+fname) and not overwrite:
continue

sys.stderr.write(f'Downloading: {url}\n')

try:
with closing(request.urlopen(url)) as r:
with open(folder+fname, 'wb') as f:
shutil.copyfileobj(r, f)
# to separete csv.xz file to decompress
if "roas.csv.xz" in url:
with closing(request.urlopen(url)) as response:
with lzma.open(BytesIO(response.read())) as r:
with open(folder+fname, 'wb') as f:
shutil.copyfileobj(r,f)
else:
with closing(request.urlopen(url)) as r:
with open(folder+fname, 'wb') as f:
shutil.copyfileobj(r, f)
except urllib.error.URLError:
sys.stderr.write(f'Error {url} is not available.\n')

4 changes: 2 additions & 2 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
readme = f.read()

setup(
version = '0.4.0',
version = '0.5.0',
name = 'rov',
author = 'Romain Fontugne',
author_email = '[email protected]',
Expand All @@ -17,7 +17,7 @@
install_requires=[
'appdirs',
'py-radix',
'portion',
'portion'
],
entry_points={'console_scripts':
['rov = rov.__main__:main']},
Expand Down