Source code for niftynet.utilities.user_parameters_parser

# -*- coding: utf-8 -*-
Parse user configuration file
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import os
import textwrap

from niftynet.engine.signal import TRAIN, INFER, EVAL
from niftynet.utilities.util_common import look_up_operations
from niftynet.engine.application_factory import ApplicationFactory
from niftynet.engine.application_factory import SUPPORTED_APP
from import resolve_file_name
from niftynet.utilities.niftynet_global_config import NiftyNetGlobalConfig
from niftynet.utilities import NiftyNetLaunchConfig
from niftynet.utilities.user_parameters_custom import SUPPORTED_ARG_SECTIONS
from niftynet.utilities.user_parameters_custom import add_customised_args
from niftynet.utilities.user_parameters_default import \
from niftynet.utilities.user_parameters_default import add_input_data_args
from niftynet.utilities.user_parameters_helper import has_section_in_config
from niftynet.utilities.user_parameters_helper import standardise_section_name
from niftynet.utilities.util_common import \
    damerau_levenshtein_distance as edit_distance
from niftynet.utilities.versioning import get_niftynet_version_string

ACTIONS = {'train': TRAIN, 'inference': INFER, 'evaluation': EVAL}
    '\n\n======\nFor more information please visit:\n' \
    '\n' \

# pylint: disable=protected-access
[docs]def available_keywords(): """ returns a list of all possible keywords defined in the parsers (duplicates from sections are removed.) """ all_key_parser = argparse.ArgumentParser( parents=[], description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, conflict_handler='resolve') for _, add_args_func in SUPPORTED_DEFAULT_SECTIONS.items(): all_key_parser = add_args_func(all_key_parser) all_key_parser = add_input_data_args(all_key_parser) # add keys from custom sections for _, add_args_func in SUPPORTED_ARG_SECTIONS.items(): all_key_parser = add_args_func(all_key_parser) default_keys = [] for action in all_key_parser._actions: try: default_keys.append(action.option_strings[0][2:]) except (IndexError, AttributeError, ValueError): pass # remove duplicates default_keys = list(set(default_keys)) # remove bad names default_keys = [keyword for keyword in default_keys if keyword] return default_keys
KEYWORDS = available_keywords() NIFTYNET_HOME = NiftyNetGlobalConfig().get_niftynet_home_folder() # pylint: disable=too-many-branches
[docs]def run(): """ meta_parser is first used to find out location of the configuration file. Based on the application_name or meta_parser.prog name, the section parsers are organised to find system parameters and application specific parameters. :return: system parameters is a group of parameters including SYSTEM_SECTIONS and app_module.REQUIRED_CONFIG_SECTION input_data_args is a group of input data sources to be used by """ meta_parser = argparse.ArgumentParser( description="Launch a NiftyNet application.", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=textwrap.dedent(EPILOG_STRING)) version_string = get_niftynet_version_string() meta_parser.add_argument("action", help="train networks, run inferences " "or evaluate inferences", metavar='ACTION', choices=list(ACTIONS)) meta_parser.add_argument("-v", "--version", action='version', version=version_string) meta_parser.add_argument("-c", "--conf", help="specify configurations from a file", metavar="CONFIG_FILE") meta_parser.add_argument("-a", "--application_name", help="specify an application module name", metavar='APPLICATION_NAME', default="") meta_args, args_from_cmdline = meta_parser.parse_known_args() print(version_string) # read configurations, to be parsed by sections config_file_name = __resolve_config_file_path(meta_args.conf) config = NiftyNetLaunchConfig()[config_file_name]) # infer application name from command app_name = None try: parser_prog = meta_parser.prog.replace('.py', '') app_name = parser_prog if parser_prog in SUPPORTED_APP \ else meta_args.application_name assert app_name except (AttributeError, AssertionError): raise ValueError( "\nUnknown application {}, or did you forget '-a' " "command argument?{}".format(app_name, EPILOG_STRING)) # load application by name app_module = ApplicationFactory.create(app_name) try: assert app_module.REQUIRED_CONFIG_SECTION, \ "\nREQUIRED_CONFIG_SECTION should be static variable " \ "in {}".format(app_module) has_section_in_config(config, app_module.REQUIRED_CONFIG_SECTION) except AttributeError: raise AttributeError( "Application code doesn't have REQUIRED_CONFIG_SECTION property. " "{} should be an instance of " "niftynet.application.base_application".format(app_module)) except ValueError: raise ValueError( "\n{} requires [{}] section in the config file.{}".format( app_name, app_module.REQUIRED_CONFIG_SECTION, EPILOG_STRING)) # check keywords in configuration file _check_config_file_keywords(config) # using configuration as default, and parsing all command line arguments # command line args override the configure file options all_args = {} for section in config.sections(): # try to rename user-specified sections for consistency section = standardise_section_name(config, section) section_defaults = dict(config.items(section)) section_args, args_from_cmdline = \ _parse_arguments_by_section([], section, section_defaults, args_from_cmdline, app_module.REQUIRED_CONFIG_SECTION) all_args[section] = section_args # check if any args from command line not recognised _check_cmd_remaining_keywords(list(args_from_cmdline)) # split parsed results in all_args # into dictionaries of system_args and input_data_args system_args, input_data_args = {}, {} for section in all_args: # copy system default sections to ``system_args`` if section in SYSTEM_SECTIONS: system_args[section] = all_args[section] continue # copy application specific sections to ``system_args`` if section == app_module.REQUIRED_CONFIG_SECTION: system_args['CUSTOM'] = all_args[section] vars(system_args['CUSTOM'])['name'] = app_name continue # copy non-default sections to ``input_data_args`` input_data_args[section] = all_args[section] # set the output path of csv list if not exists try: csv_path = resolve_file_name( input_data_args[section].csv_file, (os.path.dirname(config_file_name), NIFTYNET_HOME)) input_data_args[section].csv_file = csv_path # don't search files if csv specified in config try: delattr(input_data_args[section], 'path_to_search') except AttributeError: pass except (IOError, TypeError): input_data_args[section].csv_file = '' # preserve ``config_file`` and ``action parameter`` from the meta_args system_args['CONFIG_FILE'] = argparse.Namespace(path=config_file_name) # mapping the captured action argument to a string in ACTIONS system_args['SYSTEM'].action = \ look_up_operations(meta_args.action, ACTIONS) if not system_args['SYSTEM'].model_dir: system_args['SYSTEM'].model_dir = os.path.join( os.path.dirname(config_file_name), 'model') return system_args, input_data_args
def _parse_arguments_by_section(parents, section, args_from_config_file, args_from_cmd, required_section): """ This function first adds parameter names to a parser, according to the section name. Then it loads values from configuration files as tentative params. Finally it overrides existing pairs of 'name, value' with commandline inputs. Commandline inputs only override system/custom parameters. input data related parameters needs to be defined in config file. :param parents: a list, parsers will be created as subparsers of parents :param section: section name to be parsed :param args_from_config_file: loaded parameters from config file :param args_from_cmd: dictionary commandline parameters :return: parsed parameters of the section and unknown commandline params. """ section_parser = argparse.ArgumentParser( parents=parents, description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) try: add_args_func = SUPPORTED_DEFAULT_SECTIONS[section] except KeyError: if section == required_section: def add_args_func(parser): """ wrapper around add_customised_args """ return add_customised_args(parser, section.upper()) else: # all remaining sections are defaulting to input section add_args_func = add_input_data_args section_parser = add_args_func(section_parser) # loading all parameters a config file first if args_from_config_file is not None: section_parser.set_defaults(**args_from_config_file) # input command line input overrides config file if (section in SYSTEM_SECTIONS) or (section == required_section): section_args, unknown = section_parser.parse_known_args(args_from_cmd) return section_args, unknown # don't parse user cmd for input source sections section_args, _ = section_parser.parse_known_args([]) return section_args, args_from_cmd def _check_config_file_keywords(config): """ check config files, validate keywords provided against parsers' argument list """ # collecting all keywords from the config config_keywords = [] for section in config.sections(): if config.items(section): config_keywords.extend(list(dict(config.items(section)))) _raises_bad_keys(config_keywords, error_info='config file') def _check_cmd_remaining_keywords(args_from_cmdline): """ check list of remaining arguments from the command line input. Normally `args_from_cmd` should be empty; non-empty list means unrecognised parameters. """ args_from_cmdline = [ arg_item.replace('-', '') for arg_item in args_from_cmdline] _raises_bad_keys(args_from_cmdline, error_info='command line') # command line parameters should be valid # assertion will be triggered when keywords matched ones in custom # sections that are not used in the current application. assert not args_from_cmdline, \ '\nUnknown parameter: {}{}'.format(args_from_cmdline, EPILOG_STRING) def _raises_bad_keys(keys, error_info='config file'): """ raises value error if keys is not in the system key set. `error_info` is used to customise the error message. """ for key in list(keys): if key in KEYWORDS: continue dists = {k: edit_distance(k, key) for k in KEYWORDS} closest = min(dists, key=dists.get) raise ValueError( 'Unknown keywords in {3}: By "{0}" ' 'did you mean "{1}"?\n "{0}" is ' 'not a valid option.{2}'.format( key, closest, EPILOG_STRING, error_info)) def __resolve_config_file_path(cmdline_arg): """ Search for the absolute file name of the configuration file. starting from `-c` value provided by the user. :param cmdline_arg: :return: """ if not cmdline_arg: raise IOError("\nNo configuration file has been provided, did you " "forget '-c' command argument?{}".format(EPILOG_STRING)) # Resolve relative configuration file location config_file_path = os.path.expanduser(cmdline_arg) try: config_file_path = resolve_file_name( config_file_path, ('.', NIFTYNET_HOME)) if os.path.isfile(config_file_path): return config_file_path except (IOError, TypeError): config_file_path = os.path.expanduser(cmdline_arg) config_file_path = os.path.join( NiftyNetGlobalConfig().get_default_examples_folder(), config_file_path, config_file_path + "_config.ini") if os.path.isfile(config_file_path): return config_file_path # could not proceed without a configuration file raise IOError("\nConfiguration file not found: {}.{}".format( os.path.expanduser(cmdline_arg), EPILOG_STRING))