Skip to content

cli

BaseArgParser

Bases: ArgumentParser

An ArgumentParser specialised to support common argument handling

The 'allow_*' methods return self to permit method chaining.

Parameters:

Name Type Description Default
suppress_logs list
None
Source code in download_toolbox/cli.py 
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
class BaseArgParser(argparse.ArgumentParser):
    """An ArgumentParser specialised to support common argument handling

    The 'allow_*' methods return self to permit method chaining.

    :param suppress_logs:
    """

    def __init__(self,
                 *args,
                 log_format: str = "[%(asctime)-17s :%(levelname)-8s] - %(message)s",
                 suppress_logs: list = None,
                 **kwargs):
        super().__init__(*args, **kwargs)

        self._log_format = log_format
        self._suppress_logs = suppress_logs

        # FIXME: this is not ubiquitously useful in preprocess-toolbox: ref channel adds
        #   so there is a pattern failure having it at this level
        self.add_argument("-c", "--config-path",
                          dest="config",
                          help="Path at which to output the configuration when rendered")

        self.add_argument("-v",
                          "--verbose",
                          action="store_true",
                          default=False)

    def add_extra_args(self, extra_args):
        for arg in extra_args:
            self.add_argument(*arg[0], **arg[1])
        return self

    def parse_args(self,
                   *args,
                   **kwargs):
        args = super().parse_args(*args, **kwargs)

        self.setup_logging(verbose=args.verbose)

        return args

    def parse_known_args(self,
                         *args,
                         **kwargs):
        """
        Parses command line arguments and handles unknown arguments separately.

        This method first calls the argparse's `parse_known_args` method to separate
        known and unknown arguments. It then parses the unknown arguments using
        the `parse_known_args` function to convert the returned list to a dict.

        Args:
            *args: Variable length argument list.
            **kwargs: Arbitrary keyword arguments.

        Returns:
            A tuple containing the parsed Namespace object for known arguments
            and a dictionary of parsed unknown arguments.
        """
        args, unknown_args_list = super().parse_known_args(*args, **kwargs)

        unknown_args = parse_known_args(unknown_args_list)

        self.setup_logging(verbose=args.verbose)

        return args, unknown_args

    def setup_logging(self, verbose: bool) -> None:
        """
        Configures logging based on verbosity and suppresses logs from specific modules.

        Args:
            verbose: Whether to enable debug-level logging.
        """
        # TODO: this is not necessarily ideal when running the argparser in notebooks
        loglevel = logging.DEBUG if verbose else logging.INFO
        logging.basicConfig(
            datefmt="%d-%m-%y %T",
            format=self._log_format,
            level=loglevel
        )
        logging.getLogger().setLevel(loglevel)

        if self._suppress_logs is not None and type(self._suppress_logs) is list:
            for log_module in self._suppress_logs:
                logging.debug("Setting {} to WARNING only".format(log_module))
                logging.getLogger(log_module).setLevel(logging.WARNING)

        # TODO: bring these out of defaults
        logging.getLogger("cdsapi").setLevel(logging.WARNING)
        logging.getLogger("requests").setLevel(logging.WARNING)
        logging.getLogger("urllib3").setLevel(logging.WARNING)
        logging.getLogger("matplotlib").setLevel(logging.WARNING)

parse_known_args(*args, **kwargs)

Parses command line arguments and handles unknown arguments separately.

This method first calls the argparse's parse_known_args method to separate known and unknown arguments. It then parses the unknown arguments using the parse_known_args function to convert the returned list to a dict.

Args: args: Variable length argument list. *kwargs: Arbitrary keyword arguments.

Returns: A tuple containing the parsed Namespace object for known arguments and a dictionary of parsed unknown arguments.

Source code in download_toolbox/cli.py 
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
def parse_known_args(self,
                     *args,
                     **kwargs):
    """
    Parses command line arguments and handles unknown arguments separately.

    This method first calls the argparse's `parse_known_args` method to separate
    known and unknown arguments. It then parses the unknown arguments using
    the `parse_known_args` function to convert the returned list to a dict.

    Args:
        *args: Variable length argument list.
        **kwargs: Arbitrary keyword arguments.

    Returns:
        A tuple containing the parsed Namespace object for known arguments
        and a dictionary of parsed unknown arguments.
    """
    args, unknown_args_list = super().parse_known_args(*args, **kwargs)

    unknown_args = parse_known_args(unknown_args_list)

    self.setup_logging(verbose=args.verbose)

    return args, unknown_args

setup_logging(verbose)

Configures logging based on verbosity and suppresses logs from specific modules.

Args: verbose: Whether to enable debug-level logging.

Source code in download_toolbox/cli.py 
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
def setup_logging(self, verbose: bool) -> None:
    """
    Configures logging based on verbosity and suppresses logs from specific modules.

    Args:
        verbose: Whether to enable debug-level logging.
    """
    # TODO: this is not necessarily ideal when running the argparser in notebooks
    loglevel = logging.DEBUG if verbose else logging.INFO
    logging.basicConfig(
        datefmt="%d-%m-%y %T",
        format=self._log_format,
        level=loglevel
    )
    logging.getLogger().setLevel(loglevel)

    if self._suppress_logs is not None and type(self._suppress_logs) is list:
        for log_module in self._suppress_logs:
            logging.debug("Setting {} to WARNING only".format(log_module))
            logging.getLogger(log_module).setLevel(logging.WARNING)

    # TODO: bring these out of defaults
    logging.getLogger("cdsapi").setLevel(logging.WARNING)
    logging.getLogger("requests").setLevel(logging.WARNING)
    logging.getLogger("urllib3").setLevel(logging.WARNING)
    logging.getLogger("matplotlib").setLevel(logging.WARNING)

csv_arg(string)

Parameters:

Name Type Description Default
string str
required

Returns:

Type Description
list
Source code in download_toolbox/cli.py 
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
def csv_arg(string: str) -> list:
    """

    :param string:
    :return:
    """
    csv_items = []
    string = re.sub(r'^\'(.*)\'$', r'\1', string)

    for el in string.split(","):
        if len(el) == 0:
            csv_items.append(None)
        else:
            csv_items.append(el)
    return csv_items

csv_of_csv_arg(string)

Parameters:

Name Type Description Default
string str
required

Returns:

Type Description
list
Source code in download_toolbox/cli.py 
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
def csv_of_csv_arg(string: str) -> list:
    """

    :param string:
    :return:
    """
    csv_items = []
    string = re.sub(r'^\'(.*)\'$', r'\1', string)

    for el in string.split(","):
        if len(el) == 0:
            csv_items.append(None)
        else:
            csv_items.append(el.split("|"))
    return csv_items

csv_of_date_args(string)

Parameters:

Name Type Description Default
string str
required

Returns:

Type Description
list
Source code in download_toolbox/cli.py 
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
def csv_of_date_args(string: str) -> list:
    """

    :param string:
    :return:
    """
    csv_items = []
    string = re.sub(r'^\'(.*)\'$', r'\1', string)

    for el in string.split(","):
        if len(el) == 0:
            csv_items.append(None)
        else:
            csv_items.append([date_arg(date) for date in el.split("|")])
    return csv_items

date_arg(string)

Parameters:

Name Type Description Default
string str
required

Returns:

Type Description
object
Source code in download_toolbox/cli.py 
13
14
15
16
17
18
19
20
def date_arg(string: str) -> object:
    """

    :param string:
    :return:
    """
    date_match = re.search(r"(\d{4})-(\d{1,2})-(\d{1,2})", string)
    return dt.date(*[int(s) for s in date_match.groups()])

dates_arg(string)

Parameters:

Name Type Description Default
string str
required

Returns:

Type Description
object
Source code in download_toolbox/cli.py 
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
def dates_arg(string: str) -> object:
    """

    :param string:
    :return:
    """
    if string == "none":
        return []

    date_match = re.findall(r"(\d{4})-(\d{1,2})-(\d{1,2})", string)

    if len(date_match) < 1:
        raise argparse.ArgumentError(argument="dates",
                                     message="No dates found for supplied argument {}".format(string))
    return [dt.date(*[int(s) for s in date_tuple]) for date_tuple in date_match]

int_or_list_arg(string)

Parameters:

Name Type Description Default
string str
required

Returns:

Type Description
object
Source code in download_toolbox/cli.py 
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
def int_or_list_arg(string: str) -> object:
    """

    :param string:
    :return:
    """
    try:
        val = int(string)
    except ValueError:
        val = string.split(",")
    return val

parse_known_args(unknown_args)

Parses a list of unknown command line arguments and returns them.

This function processes both long and short options, and supports boolean flags (e.g., --flag or -f). Non-option arguments are added to the dictionary as values associated with their preceding option keys.

Args: unknown_args: A list of command line arguments.

Returns: A dictionary containing parsed arguments. Keys are converted to snake_case for consistency, with how argparse would return them.

Source code in download_toolbox/cli.py 
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
def parse_known_args(unknown_args: list) -> dict:
    """
    Parses a list of unknown command line arguments and returns them.

    This function processes both long and short options, and supports boolean flags
    (e.g., `--flag` or `-f`). Non-option arguments are added to the dictionary as
    values associated with their preceding option keys.

    Args:
        unknown_args: A list of command line arguments.

    Returns:
        A dictionary containing parsed arguments.
        Keys are converted to snake_case for consistency,
        with how argparse would return them.
    """
    result = {}
    key = None

    for arg in unknown_args:
        if arg.startswith("--"):
            # Cover long option
            if key:
                result[key] = True
            key = arg.lstrip("-").replace("-", "_")
        elif len(arg) > 1 and arg.startswith("-"):
            # Cover short option
            key = arg.replace("-", "_")
            result[key] = True
        else:
            if key:
                result[key] = arg
                key = None
            else:
                pass

    if key:
        result[key] = True

    return result