Bases: ThreadedDownloader
Downloads AMSR2 SIC data from 2012-present using HTTPS.
The data can come from yearly zips, or individual files. We target the individual files as in reality it's so much
more sensible - the zips are not consistently avialable across the data ranges
We use the following for HTTPS downloads:
- data.seaice.uni-bremen.de
But can also use (noticed when the above was down!)
- seaice.uni-bremen.de/data/
Source code in download_toolbox/data/amsr.py
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108 | class AMSRDownloader(ThreadedDownloader):
"""Downloads AMSR2 SIC data from 2012-present using HTTPS.
The data can come from yearly zips, or individual files. We target the individual files as in reality it's so much
more sensible - the zips are not consistently avialable across the data ranges
We use the following for HTTPS downloads:
- data.seaice.uni-bremen.de
But can also use (noticed when the above was down!)
- seaice.uni-bremen.de/data/
"""
def __init__(self,
dataset: AMSRDatasetConfig,
*args,
start_date: object,
host: str = "data.seaice.uni-bremen.de",
**kwargs):
amsr2_start = dt.date(2012, 7, 2)
# TODO: Differing start date ranges for different products! Validate in dataset
if start_date < amsr2_start:
raise DownloaderError("AMSR2 only exists past {}".format(amsr2_start))
self._hemi_str = "s" if dataset.location.south else "n"
self._http_client = HTTPClient("https://{}".format(host),
source_base="amsr2/asi_daygrid_swath/{}{}/netcdf".format(
self._hemi_str, "{:1.3f}".format(dataset.resolution).replace(".", "")))
super().__init__(dataset,
*args,
start_date=start_date,
**kwargs)
def _single_download(self,
var_config: object,
req_dates: object):
files_downloaded = []
for file_date in req_dates:
year_dir = str(file_date.year)
date_str = file_date.strftime("%Y%m%d")
file_in_question = "{}/asi-AMSR2-{}{}-{}-v5.4.nc".\
format(year_dir, self._hemi_str, "{:1.3f}".
format(self.dataset.resolution).
replace(".", ""), date_str)
destination_path = os.path.join(var_config.root_path, file_in_question)
if not os.path.exists(os.path.dirname(destination_path)):
os.makedirs(os.path.dirname(destination_path), exist_ok=True)
if not os.path.exists(destination_path):
try:
logging.info("Downloading {}".format(destination_path))
self._http_client.single_request(file_in_question, destination_path)
files_downloaded.append(destination_path)
except (ClientError, DownloaderError) as e:
logging.warning("Failed to download {}: {}".format(destination_path, e))
self.missing_dates.append(file_date)
else:
logging.debug("{} already exists".format(destination_path))
files_downloaded.append(destination_path)
return files_downloaded
|