From f4e8248176c798359426fd8fab6346401f0e3dbd Mon Sep 17 00:00:00 2001
From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com>
Date: Mon, 19 Jun 2023 11:54:15 +0200
Subject: [PATCH 01/11] Initial commit
---
docs/sphinx/source/reference/iotools.rst | 1 +
docs/sphinx/source/whatsnew/v0.10.0.rst | 6 ++
pvlib/iotools/__init__.py | 1 +
pvlib/iotools/srml.py | 79 +++++++++++++++++++++++-
pvlib/tests/iotools/test_srml.py | 35 ++++++++++-
5 files changed, 118 insertions(+), 4 deletions(-)
diff --git a/docs/sphinx/source/reference/iotools.rst b/docs/sphinx/source/reference/iotools.rst
index 481f46ddb5..d6dd31ac4d 100644
--- a/docs/sphinx/source/reference/iotools.rst
+++ b/docs/sphinx/source/reference/iotools.rst
@@ -17,6 +17,7 @@ of sources and file formats relevant to solar energy modeling.
iotools.parse_epw
iotools.read_srml
iotools.read_srml_month_from_solardat
+ iotools.get_srml
iotools.read_surfrad
iotools.read_midc
iotools.read_midc_raw_data_from_nrel
diff --git a/docs/sphinx/source/whatsnew/v0.10.0.rst b/docs/sphinx/source/whatsnew/v0.10.0.rst
index f7ad2b7d7c..318830178c 100644
--- a/docs/sphinx/source/whatsnew/v0.10.0.rst
+++ b/docs/sphinx/source/whatsnew/v0.10.0.rst
@@ -28,6 +28,12 @@ Enhancements
~~~~~~~~~~~~
* Added `map_variables` parameter to :py:func:`pvlib.iotools.read_srml`
and :py:func:`pvlib.iotools.read_srml_month_from_solardat` (:pull:`1773`)
+* Added :func:`pvlib.iotools.get_srml` that is similar to
+ :func:`read_srml_month_from_solardat` but is able to fetch multiple months
+ of data using the `start` and `end` parameters.
+ (:pull:`XXXX`)
+
+
Bug fixes
~~~~~~~~~
diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py
index 0a94e79f53..338f797673 100644
--- a/pvlib/iotools/__init__.py
+++ b/pvlib/iotools/__init__.py
@@ -2,6 +2,7 @@
from pvlib.iotools.epw import read_epw, parse_epw # noqa: F401
from pvlib.iotools.srml import read_srml # noqa: F401
from pvlib.iotools.srml import read_srml_month_from_solardat # noqa: F401
+from pvlib.iotools.srml import get_srml # noqa: F401
from pvlib.iotools.surfrad import read_surfrad # noqa: F401
from pvlib.iotools.midc import read_midc # noqa: F401
from pvlib.iotools.midc import read_midc_raw_data_from_nrel # noqa: F401
diff --git a/pvlib/iotools/srml.py b/pvlib/iotools/srml.py
index 6e7675482c..248f09543d 100644
--- a/pvlib/iotools/srml.py
+++ b/pvlib/iotools/srml.py
@@ -51,14 +51,14 @@ def read_srml(filename, map_variables=True):
the time of the row until the time of the next row. This is consistent
with pandas' default labeling behavior.
- See SRML's `Archival Files`_ page for more information.
-
- .. _Archival Files: http://solardat.uoregon.edu/ArchivalFiles.html
+ See [2]_ for more information concerning the file format.
References
----------
.. [1] University of Oregon Solar Radiation Monitoring Laboratory
`http://solardat.uoregon.edu/ `_
+ .. [2] `Archival (short interval) data files
+ `_
"""
tsv_data = pd.read_csv(filename, delimiter='\t')
data = format_index(tsv_data)
@@ -222,3 +222,76 @@ def read_srml_month_from_solardat(station, year, month, filetype='PO',
url = "http://solardat.uoregon.edu/download/Archive/"
data = read_srml(url + file_name, map_variables=map_variables)
return data
+
+
+def get_srml(station, start, end, filetype='PO', map_variables=True,
+ url="http://solardat.uoregon.edu/download/Archive/"):
+ """Request data from UoO SRML and read it into a Dataframe.
+
+ The Univeristy of Oregon Solar Radiation Monitoring Laboratory (SRML) is
+ described in [1]_.
+
+ Parameters
+ ----------
+ station : str
+ The name of the SRML station to request.
+ start : datetime like
+ First day of the requested period
+ end : datetime like
+ Last day of the requested period
+ filetype : string, default: 'PO'
+ SRML file type to gather. See notes for explanation.
+ map_variables : bool, default: True
+ When true, renames columns of the DataFrame to pvlib variable names
+ where applicable. See variable :const:`VARIABLE_MAP`.
+ url : str, default: 'http://solardat.uoregon.edu/download/Archive/'
+ API endpoint URL
+
+ Returns
+ -------
+ data : pd.DataFrame
+ Dataframe with data from SRML.
+ meta : dict
+ Metadata. Currently no metadata is parsed.
+
+ Notes
+ -----
+ File types designate the time interval of a file and if it contains
+ raw or processed data. For instance, `RO` designates raw, one minute
+ data and `PO` designates processed one minute data. The availability
+ of file types varies between sites. Below is a table of file types
+ and their time intervals. See [1] for site information.
+
+ ============= ============ ==================
+ time interval raw filetype processed filetype
+ ============= ============ ==================
+ 1 minute RO PO
+ 5 minute RF PF
+ 15 minute RQ PQ
+ hourly RH PH
+ ============= ============ ==================
+
+ References
+ ----------
+ .. [1] University of Oregon Solar Radiation Measurement Laboratory
+ `http://solardat.uoregon.edu/ `_
+ """
+ # Use pd.to_datetime so that strings (e.g. '2021-01-01') are accepted
+ start = pd.to_datetime(start)
+ end = pd.to_datetime(end)
+
+ # Generate list of months
+ months = pd.date_range(
+ start, end.replace(day=1) + pd.DateOffset(months=1), freq='1M')
+ months_str = months.strftime('%y%m')
+
+ # Generate list of filenames
+ filenames = [f"{station}{filetype}{m}.txt" for m in months_str]
+
+ dfs = [read_srml(url + f, map_variables=map_variables) for f in filenames]
+
+ data = pd.concat(dfs, axis='rows')
+
+ meta = {}
+
+ return data, meta
diff --git a/pvlib/tests/iotools/test_srml.py b/pvlib/tests/iotools/test_srml.py
index 9939374ebf..26f96a9a4e 100644
--- a/pvlib/tests/iotools/test_srml.py
+++ b/pvlib/tests/iotools/test_srml.py
@@ -3,7 +3,8 @@
import pytest
from pvlib.iotools import srml
-from ..conftest import DATA_DIR, RERUNS, RERUNS_DELAY
+from ..conftest import (DATA_DIR, RERUNS, RERUNS_DELAY, assert_index_equal,
+ assert_frame_equal)
srml_testfile = DATA_DIR / 'SRML-day-EUPO1801.txt'
@@ -107,3 +108,35 @@ def test_hourly_dt_index():
assert data.index[0] == start
assert data.index[-1] == end
assert (data.index.minute == 0).all()
+
+
+@pytest.fixture
+def index_hourly():
+ return pd.date_range(start='1986-04-01', end='1986-05-31 23:59',
+ freq='1h', tz='Etc/GMT+8')
+
+
+@pytest.mark.remote_data
+@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
+def test_get_srml_hourly(index_hourly):
+ data, meta = data, meta = srml.get_srml(station='CD', start='1986-04-01',
+ end='1986-05-31', filetype='PH')
+ assert_index_equal(data.index, index_hourly)
+
+
+@pytest.fixture
+def index_minute():
+ return pd.date_range(start='2018-01-01', end='2018-01-31 23:59',
+ freq='1min', tz='Etc/GMT+8')
+
+
+@pytest.mark.remote_data
+@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
+def test_get_srml_minute(index_minute):
+ data_read = srml.read_srml(srml_testfile)
+ data_get, meta = srml.get_srml(station='EU', start='2018-01-01',
+ end='2018-01-31', filetype='PO')
+ assert_index_equal(data_get.index, index_minute)
+ assert all([c in data_get.columns for c in data_read.columns])
+ # Check that all indices in example file are present in remote file
+ assert data_read.index.isin(data_get.index).all()
From 0f83b2eed9ae49551afe999601ef799a2d134418 Mon Sep 17 00:00:00 2001
From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com>
Date: Mon, 19 Jun 2023 12:00:35 +0200
Subject: [PATCH 02/11] Add exception for non-existing file names
---
pvlib/iotools/srml.py | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/pvlib/iotools/srml.py b/pvlib/iotools/srml.py
index 248f09543d..c4107447f9 100644
--- a/pvlib/iotools/srml.py
+++ b/pvlib/iotools/srml.py
@@ -3,6 +3,8 @@
"""
import numpy as np
import pandas as pd
+import requests
+import warnings
# VARIABLE_MAP is a dictionary mapping SRML data element numbers to their
@@ -229,12 +231,12 @@ def get_srml(station, start, end, filetype='PO', map_variables=True,
"""Request data from UoO SRML and read it into a Dataframe.
The Univeristy of Oregon Solar Radiation Monitoring Laboratory (SRML) is
- described in [1]_.
+ described in [1]_. A list of stations can be found in [2]_.
Parameters
----------
station : str
- The name of the SRML station to request.
+ Two letter station abbreviation.
start : datetime like
First day of the requested period
end : datetime like
@@ -275,6 +277,8 @@ def get_srml(station, start, end, filetype='PO', map_variables=True,
----------
.. [1] University of Oregon Solar Radiation Measurement Laboratory
`http://solardat.uoregon.edu/ `_
+ .. [2] `Station ID codes - Solar Radiation Measurement Laboratory
+ http://solardat.uoregon.edu/StationIDCodes.html>`_
"""
# Use pd.to_datetime so that strings (e.g. '2021-01-01') are accepted
start = pd.to_datetime(start)
@@ -288,7 +292,13 @@ def get_srml(station, start, end, filetype='PO', map_variables=True,
# Generate list of filenames
filenames = [f"{station}{filetype}{m}.txt" for m in months_str]
- dfs = [read_srml(url + f, map_variables=map_variables) for f in filenames]
+ dfs = [] # Initialize list of monthly dataframes
+ for f in filenames:
+ try:
+ dfi = read_srml(url + f, map_variables=map_variables)
+ dfs.append(dfi)
+ except requests.HTTPError:
+ warnings.warn(f"The following file was not found: {f}")
data = pd.concat(dfs, axis='rows')
From 8f14cb86b6ec2f19f14315afd7d54bf4209a9c9b Mon Sep 17 00:00:00 2001
From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com>
Date: Mon, 19 Jun 2023 12:06:49 +0200
Subject: [PATCH 03/11] Update error handling
---
pvlib/iotools/srml.py | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/pvlib/iotools/srml.py b/pvlib/iotools/srml.py
index c4107447f9..55192337e3 100644
--- a/pvlib/iotools/srml.py
+++ b/pvlib/iotools/srml.py
@@ -3,7 +3,7 @@
"""
import numpy as np
import pandas as pd
-import requests
+import urllib
import warnings
@@ -28,8 +28,9 @@
def read_srml(filename, map_variables=True):
"""
- Read University of Oregon SRML 1min .tsv file into pandas dataframe. The
- SRML is described in [1]_.
+ Read University of Oregon SRML 1min .tsv file into pandas dataframe.
+
+ The SRML is described in [1]_.
Parameters
----------
@@ -172,8 +173,9 @@ def format_index(df):
def read_srml_month_from_solardat(station, year, month, filetype='PO',
map_variables=True):
- """Request a month of SRML data from solardat and read it into
- a Dataframe. The SRML is described in [1]_.
+ """Request a month of SRML data and read it into a Dataframe.
+
+ The SRML is described in [1]_.
Parameters
----------
@@ -297,7 +299,7 @@ def get_srml(station, start, end, filetype='PO', map_variables=True,
try:
dfi = read_srml(url + f, map_variables=map_variables)
dfs.append(dfi)
- except requests.HTTPError:
+ except urllib.error.HTTPError:
warnings.warn(f"The following file was not found: {f}")
data = pd.concat(dfs, axis='rows')
From b09144f6a54dfa6db2d3291b8338f39103796577 Mon Sep 17 00:00:00 2001
From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com>
Date: Mon, 19 Jun 2023 17:19:06 +0200
Subject: [PATCH 04/11] Deprecate read_srml_month_from_solardat
---
pvlib/iotools/srml.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/pvlib/iotools/srml.py b/pvlib/iotools/srml.py
index 55192337e3..25d259a943 100644
--- a/pvlib/iotools/srml.py
+++ b/pvlib/iotools/srml.py
@@ -6,6 +6,7 @@
import urllib
import warnings
+from pvlib._deprecation import deprecated
# VARIABLE_MAP is a dictionary mapping SRML data element numbers to their
# pvlib names. For most variables, only the first three digits are used,
@@ -171,6 +172,7 @@ def format_index(df):
return df
+@deprecated('0.10.0', alternative='pvlib.iotools.get_srml')
def read_srml_month_from_solardat(station, year, month, filetype='PO',
map_variables=True):
"""Request a month of SRML data and read it into a Dataframe.
From 7057ede8909f2c488447f55555431eac9ec05086 Mon Sep 17 00:00:00 2001
From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com>
Date: Mon, 19 Jun 2023 17:58:51 +0200
Subject: [PATCH 05/11] Add fail_on_pvlib_version to tests
---
pvlib/iotools/srml.py | 7 ++++---
pvlib/tests/iotools/test_srml.py | 9 ++++++---
2 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/pvlib/iotools/srml.py b/pvlib/iotools/srml.py
index 25d259a943..a4fd0e9c73 100644
--- a/pvlib/iotools/srml.py
+++ b/pvlib/iotools/srml.py
@@ -172,7 +172,7 @@ def format_index(df):
return df
-@deprecated('0.10.0', alternative='pvlib.iotools.get_srml')
+@deprecated('0.10.0', alternative='pvlib.iotools.get_srml', removal='0.11.0')
def read_srml_month_from_solardat(station, year, month, filetype='PO',
map_variables=True):
"""Request a month of SRML data and read it into a Dataframe.
@@ -281,8 +281,9 @@ def get_srml(station, start, end, filetype='PO', map_variables=True,
----------
.. [1] University of Oregon Solar Radiation Measurement Laboratory
`http://solardat.uoregon.edu/ `_
- .. [2] `Station ID codes - Solar Radiation Measurement Laboratory
- http://solardat.uoregon.edu/StationIDCodes.html>`_
+ .. [2] Station ID codes - Solar Radiation Measurement Laboratory
+ `http://solardat.uoregon.edu/StationIDCodes.html
+ `_
"""
# Use pd.to_datetime so that strings (e.g. '2021-01-01') are accepted
start = pd.to_datetime(start)
diff --git a/pvlib/tests/iotools/test_srml.py b/pvlib/tests/iotools/test_srml.py
index 26f96a9a4e..0225a2bd7f 100644
--- a/pvlib/tests/iotools/test_srml.py
+++ b/pvlib/tests/iotools/test_srml.py
@@ -4,7 +4,7 @@
from pvlib.iotools import srml
from ..conftest import (DATA_DIR, RERUNS, RERUNS_DELAY, assert_index_equal,
- assert_frame_equal)
+ assert_frame_equal, fail_on_pvlib_version)
srml_testfile = DATA_DIR / 'SRML-day-EUPO1801.txt'
@@ -77,13 +77,15 @@ def test_map_columns(column, expected):
@pytest.mark.remote_data
@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
-def test_read_srml_month_from_solardat():
+def test_get_srml():
url = 'http://solardat.uoregon.edu/download/Archive/EUPO1801.txt'
file_data = srml.read_srml(url)
- requested = srml.read_srml_month_from_solardat('EU', 2018, 1)
+ requested, _ = srml.get_srml(station='EU', start='2018-01-01',
+ end='2018-01-31')
assert file_data.equals(requested)
+@fail_on_pvlib_version('0.11')
@pytest.mark.remote_data
@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
def test_15_minute_dt_index():
@@ -97,6 +99,7 @@ def test_15_minute_dt_index():
assert (data.index[3::4].minute == 45).all()
+@fail_on_pvlib_version('0.11')
@pytest.mark.remote_data
@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
def test_hourly_dt_index():
From 2b5c39b3fb705fccbf771e2fa5ec79204d1f5d5d Mon Sep 17 00:00:00 2001
From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com>
Date: Thu, 22 Jun 2023 00:38:49 +0200
Subject: [PATCH 06/11] Apply suggestions from code review
Co-authored-by: Kevin Anderson
---
docs/sphinx/source/whatsnew/v0.10.0.rst | 4 ++--
pvlib/iotools/srml.py | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/docs/sphinx/source/whatsnew/v0.10.0.rst b/docs/sphinx/source/whatsnew/v0.10.0.rst
index 318830178c..6333158ac3 100644
--- a/docs/sphinx/source/whatsnew/v0.10.0.rst
+++ b/docs/sphinx/source/whatsnew/v0.10.0.rst
@@ -29,9 +29,9 @@ Enhancements
* Added `map_variables` parameter to :py:func:`pvlib.iotools.read_srml`
and :py:func:`pvlib.iotools.read_srml_month_from_solardat` (:pull:`1773`)
* Added :func:`pvlib.iotools.get_srml` that is similar to
- :func:`read_srml_month_from_solardat` but is able to fetch multiple months
+ :func:`pvlib.iotools.read_srml_month_from_solardat` but is able to fetch multiple months
of data using the `start` and `end` parameters.
- (:pull:`XXXX`)
+ (:pull:`1779`)
diff --git a/pvlib/iotools/srml.py b/pvlib/iotools/srml.py
index a4fd0e9c73..20dc77a39d 100644
--- a/pvlib/iotools/srml.py
+++ b/pvlib/iotools/srml.py
@@ -234,7 +234,7 @@ def get_srml(station, start, end, filetype='PO', map_variables=True,
url="http://solardat.uoregon.edu/download/Archive/"):
"""Request data from UoO SRML and read it into a Dataframe.
- The Univeristy of Oregon Solar Radiation Monitoring Laboratory (SRML) is
+ The University of Oregon Solar Radiation Monitoring Laboratory (SRML) is
described in [1]_. A list of stations can be found in [2]_.
Parameters
From d689a147da6e57a14b4fee1911d6bd3f260ec337 Mon Sep 17 00:00:00 2001
From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com>
Date: Thu, 22 Jun 2023 00:52:54 +0200
Subject: [PATCH 07/11] Address code review
---
docs/sphinx/source/whatsnew/v0.9.6.rst | 4 ++-
pvlib/iotools/srml.py | 4 ++-
pvlib/tests/iotools/test_srml.py | 39 +++++++++++++++-----------
3 files changed, 28 insertions(+), 19 deletions(-)
diff --git a/docs/sphinx/source/whatsnew/v0.9.6.rst b/docs/sphinx/source/whatsnew/v0.9.6.rst
index 3ec9bf9f9b..17b35d9619 100644
--- a/docs/sphinx/source/whatsnew/v0.9.6.rst
+++ b/docs/sphinx/source/whatsnew/v0.9.6.rst
@@ -28,11 +28,13 @@ Deprecations
(data period 2003-2012). Instead, ECMWF recommends to use CAMS global
reanalysis (EAC4) from the Atmosphere Data Store (ADS). See also :py:func:`pvlib.iotools.get_cams`.
(:issue:`1691`, :pull:`1654`)
-
* The ``recolumn`` parameter in :py:func:`pvlib.iotools.read_tmy3`, which maps
TMY3 column names to nonstandard alternatives, is now deprecated.
We encourage using ``map_variables`` (which produces standard pvlib names) instead.
(:issue:`1517`, :pull:`1623`)
+* :py:func:`pvlib.iotools.read_srml_month_from_solardat` is deprecated and replaced by
+ :py:func:`pvlib.iotools.get_srml`. (:pull:`1779`)
+
Enhancements
~~~~~~~~~~~~
diff --git a/pvlib/iotools/srml.py b/pvlib/iotools/srml.py
index 20dc77a39d..51be901378 100644
--- a/pvlib/iotools/srml.py
+++ b/pvlib/iotools/srml.py
@@ -307,6 +307,8 @@ def get_srml(station, start, end, filetype='PO', map_variables=True,
data = pd.concat(dfs, axis='rows')
- meta = {}
+ meta = {'filetype': filetype,
+ 'station': station,
+ 'filenames': filenames}
return data, meta
diff --git a/pvlib/tests/iotools/test_srml.py b/pvlib/tests/iotools/test_srml.py
index 0225a2bd7f..f2fe7a49b5 100644
--- a/pvlib/tests/iotools/test_srml.py
+++ b/pvlib/tests/iotools/test_srml.py
@@ -4,7 +4,7 @@
from pvlib.iotools import srml
from ..conftest import (DATA_DIR, RERUNS, RERUNS_DELAY, assert_index_equal,
- assert_frame_equal, fail_on_pvlib_version)
+ fail_on_pvlib_version)
srml_testfile = DATA_DIR / 'SRML-day-EUPO1801.txt'
@@ -85,6 +85,16 @@ def test_get_srml():
assert file_data.equals(requested)
+@fail_on_pvlib_version('0.11')
+@pytest.mark.remote_data
+@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
+def test_read_srml_month_from_solardat():
+ url = 'http://solardat.uoregon.edu/download/Archive/EUPO1801.txt'
+ file_data = srml.read_srml(url)
+ requested = srml.read_srml_month_from_solardat('EU', 2018, 1)
+ assert file_data.equals(requested)
+
+
@fail_on_pvlib_version('0.11')
@pytest.mark.remote_data
@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
@@ -113,33 +123,28 @@ def test_hourly_dt_index():
assert (data.index.minute == 0).all()
-@pytest.fixture
-def index_hourly():
- return pd.date_range(start='1986-04-01', end='1986-05-31 23:59',
- freq='1h', tz='Etc/GMT+8')
-
-
@pytest.mark.remote_data
@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
-def test_get_srml_hourly(index_hourly):
+def test_get_srml_hourly():
data, meta = data, meta = srml.get_srml(station='CD', start='1986-04-01',
end='1986-05-31', filetype='PH')
- assert_index_equal(data.index, index_hourly)
-
-
-@pytest.fixture
-def index_minute():
- return pd.date_range(start='2018-01-01', end='2018-01-31 23:59',
- freq='1min', tz='Etc/GMT+8')
+ expected_index = pd.date_range(start='1986-04-01', end='1986-05-31 23:59',
+ freq='1h', tz='Etc/GMT+8')
+ assert_index_equal(data.index, expected_index)
@pytest.mark.remote_data
@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
-def test_get_srml_minute(index_minute):
+def test_get_srml_minute():
data_read = srml.read_srml(srml_testfile)
data_get, meta = srml.get_srml(station='EU', start='2018-01-01',
end='2018-01-31', filetype='PO')
- assert_index_equal(data_get.index, index_minute)
+ expected_index = pd.date_range(start='2018-01-01', end='2018-01-31 23:59',
+ freq='1min', tz='Etc/GMT+8')
+ assert_index_equal(data_get.index, expected_index)
assert all([c in data_get.columns for c in data_read.columns])
# Check that all indices in example file are present in remote file
assert data_read.index.isin(data_get.index).all()
+ assert meta['station'] == 'EU'
+ assert meta['filetype'] == 'PO'
+ assert meta['filenames'] == ['EUPO1801.txt']
From e85ddf78369b0a0a16f705e080bc7fbd5d78a280 Mon Sep 17 00:00:00 2001
From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com>
Date: Sun, 25 Jun 2023 22:45:22 +0200
Subject: [PATCH 08/11] Conform to code review
---
pvlib/iotools/srml.py | 4 +++-
pvlib/tests/iotools/test_srml.py | 20 +++++++++++++++++---
2 files changed, 20 insertions(+), 4 deletions(-)
diff --git a/pvlib/iotools/srml.py b/pvlib/iotools/srml.py
index 51be901378..d1f6cf4f92 100644
--- a/pvlib/iotools/srml.py
+++ b/pvlib/iotools/srml.py
@@ -237,6 +237,8 @@ def get_srml(station, start, end, filetype='PO', map_variables=True,
The University of Oregon Solar Radiation Monitoring Laboratory (SRML) is
described in [1]_. A list of stations can be found in [2]_.
+ Data is returned for the entire months between and including start and end.
+
Parameters
----------
station : str
@@ -258,7 +260,7 @@ def get_srml(station, start, end, filetype='PO', map_variables=True,
data : pd.DataFrame
Dataframe with data from SRML.
meta : dict
- Metadata. Currently no metadata is parsed.
+ Metadata.
Notes
-----
diff --git a/pvlib/tests/iotools/test_srml.py b/pvlib/tests/iotools/test_srml.py
index f2fe7a49b5..4d62529ff6 100644
--- a/pvlib/tests/iotools/test_srml.py
+++ b/pvlib/tests/iotools/test_srml.py
@@ -5,6 +5,7 @@
from pvlib.iotools import srml
from ..conftest import (DATA_DIR, RERUNS, RERUNS_DELAY, assert_index_equal,
fail_on_pvlib_version)
+from pvlib._deprecation import pvlibDeprecationWarning
srml_testfile = DATA_DIR / 'SRML-day-EUPO1801.txt'
@@ -91,7 +92,8 @@ def test_get_srml():
def test_read_srml_month_from_solardat():
url = 'http://solardat.uoregon.edu/download/Archive/EUPO1801.txt'
file_data = srml.read_srml(url)
- requested = srml.read_srml_month_from_solardat('EU', 2018, 1)
+ with pytest.warns(pvlibDeprecationWarning, match='get_srml instead'):
+ requested = srml.read_srml_month_from_solardat('EU', 2018, 1)
assert file_data.equals(requested)
@@ -99,7 +101,8 @@ def test_read_srml_month_from_solardat():
@pytest.mark.remote_data
@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
def test_15_minute_dt_index():
- data = srml.read_srml_month_from_solardat('TW', 2019, 4, 'RQ')
+ with pytest.warns(pvlibDeprecationWarning, match='get_srml instead'):
+ data = srml.read_srml_month_from_solardat('TW', 2019, 4, 'RQ')
start = pd.Timestamp('20190401 00:00')
start = start.tz_localize('Etc/GMT+8')
end = pd.Timestamp('20190430 23:45')
@@ -113,7 +116,8 @@ def test_15_minute_dt_index():
@pytest.mark.remote_data
@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
def test_hourly_dt_index():
- data = srml.read_srml_month_from_solardat('CD', 1986, 4, 'PH')
+ with pytest.warns(pvlibDeprecationWarning, match='get_srml instead'):
+ data = srml.read_srml_month_from_solardat('CD', 1986, 4, 'PH')
start = pd.Timestamp('19860401 00:00')
start = start.tz_localize('Etc/GMT+8')
end = pd.Timestamp('19860430 23:00')
@@ -148,3 +152,13 @@ def test_get_srml_minute():
assert meta['station'] == 'EU'
assert meta['filetype'] == 'PO'
assert meta['filenames'] == ['EUPO1801.txt']
+
+
+@pytest.mark.remote_data
+@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY)
+def test_get_srml_nonexisting_month_warning():
+ with pytest.warns(UserWarning, match='file was not found: EUPO0912.txt'):
+ # Request data for a period where not all files exist
+ # Eugene (EU) station started reporting 1-minute data in January 2010
+ data, meta = data, meta = srml.get_srml(
+ station='EU', start='2009-12-01', end='2010-01-31', filetype='PO')
From a308cda05a6e0d48b91d31d44b553f580bb8234f Mon Sep 17 00:00:00 2001
From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com>
Date: Wed, 28 Jun 2023 13:57:11 +0200
Subject: [PATCH 09/11] Add warning message
---
pvlib/iotools/srml.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/pvlib/iotools/srml.py b/pvlib/iotools/srml.py
index f78c322701..81b61556a0 100644
--- a/pvlib/iotools/srml.py
+++ b/pvlib/iotools/srml.py
@@ -279,6 +279,11 @@ def get_srml(station, start, end, filetype='PO', map_variables=True,
hourly RH PH
============= ============ ==================
+ Warning
+ -------
+ SRML data has nighttime data prefilled with 0s through the end of the
+ current month (i.e., values are provided for data in the future).
+
References
----------
.. [1] University of Oregon Solar Radiation Measurement Laboratory
From 1830b03631528c1d5db276f5c7013a8b776675d2 Mon Sep 17 00:00:00 2001
From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com>
Date: Wed, 28 Jun 2023 13:57:24 +0200
Subject: [PATCH 10/11] Use assert_frame_equal
---
pvlib/tests/iotools/test_srml.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pvlib/tests/iotools/test_srml.py b/pvlib/tests/iotools/test_srml.py
index e225c429c9..147ec3e3c3 100644
--- a/pvlib/tests/iotools/test_srml.py
+++ b/pvlib/tests/iotools/test_srml.py
@@ -4,7 +4,7 @@
from pvlib.iotools import srml
from ..conftest import (DATA_DIR, RERUNS, RERUNS_DELAY, assert_index_equal,
- fail_on_pvlib_version)
+ assert_frame_equal, fail_on_pvlib_version)
from pvlib._deprecation import pvlibDeprecationWarning
srml_testfile = DATA_DIR / 'SRML-day-EUPO1801.txt'
@@ -83,7 +83,7 @@ def test_get_srml():
file_data = srml.read_srml(url)
requested, _ = srml.get_srml(station='EU', start='2018-01-01',
end='2018-01-31')
- assert file_data.equals(requested)
+ assert assert_frame_equal(file_data, requested)
@fail_on_pvlib_version('0.11')
From a1b39226827a840596d8a33abc1fad5424df3c43 Mon Sep 17 00:00:00 2001
From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com>
Date: Thu, 29 Jun 2023 16:24:08 +0200
Subject: [PATCH 11/11] Update pvlib/tests/iotools/test_srml.py
Co-authored-by: Kevin Anderson
---
pvlib/tests/iotools/test_srml.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pvlib/tests/iotools/test_srml.py b/pvlib/tests/iotools/test_srml.py
index 147ec3e3c3..8f960885d4 100644
--- a/pvlib/tests/iotools/test_srml.py
+++ b/pvlib/tests/iotools/test_srml.py
@@ -83,7 +83,7 @@ def test_get_srml():
file_data = srml.read_srml(url)
requested, _ = srml.get_srml(station='EU', start='2018-01-01',
end='2018-01-31')
- assert assert_frame_equal(file_data, requested)
+ assert_frame_equal(file_data, requested)
@fail_on_pvlib_version('0.11')