"""cfgparse - a powerful, extensible, and easy-to-use configuration parser. By Dan Gass If you have problems with this module, please file bugs through the Source Forge project page: http://sourceforge.net/projects/cfgparse """ # @future use option note when get error # @future print file/section/linenumber information when checks fail # @future - check type='choice' and choices=[] one must have the other # @future -- do we have a command line --cfgcheck option that expands all configuration and checks all possible keys? __version__ = "1.00" __all__ = [] __copyright__ = """ Copyright (c) 2004 by Daniel M. Gass. All rights reserved. Copyright (c) 2001-2004 Gregory P. Ward. All rights reserved. Copyright (c) 2002-2004 Python Software Foundation. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ import ConfigParser as _ConfigParser import cStringIO import os import re import sys import textwrap # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # U T I L I T Y # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # try: from gettext import gettext as _ except ImportError: _ = lambda arg: arg class HelpFormatter: """ Abstract base class for formatting option help. ConfigParser instances should use one of the HelpFormatter subclasses for formatting help; by default IndentedHelpFormatter is used. Instance attributes: parser : OptionParser the controlling OptionParser instance indent_increment : int the number of columns to indent per nesting level max_help_position : int the maximum starting column for option help text help_position : int the calculated starting column for option help text; initially the same as the maximum width : int total number of columns for output (pass None to constructor for this value to be taken from the $COLUMNS environment variable) level : int current indentation level current_indent : int current indentation level (in columns) help_width : int number of columns available for option help text (calculated) default_tag : str text to replace with each option's default value, "%default" by default. Set to false value to disable default value expansion. option_strings : { Option : str } maps Option instances to the snippet of help text explaining the syntax of that option, e.g. "option=VALUE" """ NO_DEFAULT_VALUE = "none" def __init__(self, indent_increment, max_help_position, width, short_first): self.parser = None self.indent_increment = indent_increment self.help_position = self.max_help_position = max_help_position if width is None: try: width = int(os.environ['COLUMNS']) except (KeyError, ValueError): width = 80 width -= 2 self.width = width self.current_indent = 0 self.level = 0 self.help_width = None # computed later self.short_first = short_first self.default_tag = "%default" self.option_strings = {} self._short_opt_fmt = "%s %s" self._long_opt_fmt = "%s=%s" def set_parser(self, parser): self.parser = parser def indent(self): self.current_indent += self.indent_increment self.level += 1 def dedent(self): self.current_indent -= self.indent_increment assert self.current_indent >= 0, "Indent decreased below 0." self.level -= 1 def format_usage(self, usage): raise NotImplementedError, "subclasses must implement" def format_heading(self, heading): raise NotImplementedError, "subclasses must implement" def format_description(self, description): if not description: return "" desc_width = self.width - self.current_indent indent = " "*self.current_indent return textwrap.fill(description, desc_width, initial_indent=indent, subsequent_indent=indent) + "\n" def expand_default(self, option): if self.parser is None or not self.default_tag: return option.help try: default_value = option.default except AttributeError: default_value = None if default_value is NO_DEFAULT or default_value is None: default_value = self.NO_DEFAULT_VALUE return option.help.replace(self.default_tag, str(default_value)) def format_option(self, option): # The help for each option consists of two parts: # * the opt strings and metavars # eg. ("option=VALUE") # * the user-supplied help string # eg. ("turn on expert mode", "read data from FILENAME") # # If possible, we write both of these on the same line: # option=VALUE explanation of some option # # But if the opt string list is too long, we put the help # string on a second line, indented to the same column it would # start in if it fit on the first line. # thisoption=WAY_TO_LONG # explanation of the long option result = [] opts = self.option_strings[option] opt_width = self.help_position - self.current_indent - 2 if len(opts) > opt_width: opts = "%*s%s\n" % (self.current_indent, "", opts) indent_first = self.help_position else: # start help on same line as opts opts = "%*s%-*s " % (self.current_indent, "", opt_width, opts) indent_first = 0 result.append(opts) if option.help: help_text = self.expand_default(option) help_lines = textwrap.wrap(help_text, self.help_width) result.append("%*s%s\n" % (indent_first, "", help_lines[0])) result.extend(["%*s%s\n" % (self.help_position, "", line) for line in help_lines[1:]]) elif opts[-1] != "\n": result.append("\n") return "".join(result) def store_option_strings(self, parser): self.indent() max_len = 0 for opt in parser.option_list: strings = self.format_option_strings(opt) self.option_strings[opt] = strings max_len = max(max_len, len(strings) + self.current_indent) self.indent() for group in parser.option_groups: for opt in group.option_list: strings = self.format_option_strings(opt) self.option_strings[opt] = strings max_len = max(max_len, len(strings) + self.current_indent) self.dedent() self.dedent() self.help_position = min(max_len + 2, self.max_help_position) self.help_width = self.width - self.help_position def format_option_strings(self, option): """Return a comma-separated list of option strings & metavariables.""" metavar = option.metavar or option.dest.upper() return '%s=%s' % (option.name,metavar) class IndentedHelpFormatter (HelpFormatter): """Format help with indented section bodies. """ # NOTE optparse def __init__(self, indent_increment=2, max_help_position=24, width=None, short_first=1): HelpFormatter.__init__( self, indent_increment, max_help_position, width, short_first) def format_usage(self, usage): return _("usage: %s\n") % usage def format_heading(self, heading): return "%*s%s:\n" % (self.current_indent, "", heading) class TitledHelpFormatter (HelpFormatter): """Format help with underlined section headers. """ # NOTE optparse def __init__(self, indent_increment=0, max_help_position=24, width=None, short_first=0): HelpFormatter.__init__ ( self, indent_increment, max_help_position, width, short_first) def format_usage(self, usage): return "%s %s\n" % (self.format_heading(_("Usage")), usage) def format_heading(self, heading): return "%s\n%s\n" % (heading, "=-"[self.level] * len(heading)) SUPPRESS_HELP = "SUPPRESS"+"HELP" NO_DEFAULT = ("NO", "DEFAULT") def _repr(self): return "<%s at 0x%x: %s>" % (self.__class__.__name__, id(self), self) # NOTHING_FOUND = ("NOTHING","FOUND") def split_keys(keys): """Returns list of keys resulting from keys argument. --- NO KEYS --- >>> split_keys( None ) [] >>> split_keys( [] ) [] --- STRINGS --- >>> split_keys( "key1" ) ['key1'] >>> split_keys( "key1,key2" ) ['key1', 'key2'] >>> split_keys( "key1.key2" ) ['key1', 'key2'] >>> split_keys( "key1.key2,key3" ) ['key1', 'key2', 'key3'] --- LISTS --- >>> split_keys( ['key1'] ) # single item ['key1'] >>> split_keys( ['key1','key2'] ) # multiple items ['key1', 'key2'] --- QUOTING --- These tests check that quotes can be used to protect '.' and ','. >>> split_keys( "'some.key1','some,key2'" ) # single ticks work ['some.key1', 'some,key2'] >>> split_keys( '"some,key1","some.key2"' ) # double ticks work ['some,key1', 'some.key2'] >>> split_keys( '"some,key1",some.key2' ) # must quote everything Traceback (most recent call last): ConfigParserError: Keys not quoted properly. Quotes must surround all keys. >>> split_keys( "some,key1,'some.key2'" ) # must quote everything Traceback (most recent call last): ConfigParserError: Keys not quoted properly. Quotes must surround all keys. >>> split_keys( "key1,'some.key2'.key3" ) # must quote everything Traceback (most recent call last): ConfigParserError: Keys not quoted properly. Quotes must surround all keys. >>> split_keys('DEFAULT') [] >>> split_keys(['DEFAULT']) [] """ if (keys is None) or (keys == 'DEFAULT') or (keys == ['DEFAULT']): return [] try: keys = keys.replace('"',"'") if "'" in keys: keys = keys.strip("'").split("','") for key in keys: if "'" in key: IMPROPER_QUOTES = "Keys not quoted properly. Quotes must surround all keys." raise ConfigParserUserError(IMPROPER_QUOTES) else: keys = keys.replace('.',',').split(',') except AttributeError: pass return keys def join_keys(keys,sep=','): """ >>> join_keys(['key1']) 'key1' >>> join_keys(['key1','key2']) 'key1,key2' >>> join_keys(['key1','key2'],'.') 'key1.key2' >>> join_keys(['key.1','key2'],'.') "'key.1','key2'" >>> join_keys(['key,1','key2'],'.') "'key,1','key2'" >>> join_keys([]) 'DEFAULT' """ if not keys: return 'DEFAULT' mash = ''.join(keys) if '.' in mash or ',' in mash: quote = "'" sep = quote + ',' + quote return quote + sep.join(keys) + quote return sep.join(keys) def split_paths(paths): """Returns list of paths resulting from paths argument. --- NO KEYS --- >>> split_paths( None ) [] >>> split_paths( [] ) [] --- STRINGS --- >>> split_paths( "path1" ) ['path1'] >>> split_paths( os.path.pathsep.join(["path1","path2"]) ) ['path1', 'path2'] >>> split_paths( "path.1,path.2" ) ['path.1', 'path.2'] --- LISTS --- >>> split_paths( ['path1'] ) # single item ['path1'] >>> split_paths( ['path1','path2'] ) # multiple items ['path1', 'path2'] """ if paths is None: return [] try: return paths.replace(',',os.path.pathsep).split(os.path.pathsep) except AttributeError: return paths # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # E X C E P T I O N S # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # class ConfigParserError(Exception): """Exception associated with the cfgparse module""" pass class ConfigParserAppError(Exception): """Exception due to application programming error""" pass class ConfigParserUserError(Exception): """Exception due to user error""" pass # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # K E Y S # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # class Keys(object): """Prioritizes and stores default configuration keys. This class is used to store default configuration keys for an instance of the Config class. The different sources of keys are supported (stored) by the following methods of this class: add_cfg_keys -- configuration file specified default keys add_cmd_keys -- command line option keys add_env_keys -- environment variable keys The 'get' method returns a combined list of keys in the following order: application keys (passed when calling 'get' method) command line keys environment variable keys configuration file keys a 'DEFAULT' key (always present) Setup: modify os module to fake out environment variable gets >>> _getenv = os.getenv >>> def getenv(variable,default): ... if variable == 'VAR12': ... return 'env1,env2' ... elif variable == 'VAR3': ... return 'env3' ... elif variable == 'VAR4': ... return 'env4' ... else: ... return default >>> os.getenv = getenv Case 1: normal string lists of keys >>> k = Keys() >>> k.add_env_keys('VAR12,VAR3') # this has effect >>> k.add_env_keys('VAR_NONE') # no effect >>> k.add_cfg_keys('cfg1,cfg2') >>> k.add_cmd_keys('cmd1.cmd2') >>> k.add_env_keys('VAR12') # no effect (already read) >>> k.get('app1,app2') ['app1', 'app2', 'cmd1', 'cmd2', 'env1', 'env2', 'env3', 'cfg1', 'cfg2', 'DEFAULT'] Case 2: extend lists using other key input flavors just to make sure each method uses split_keys() >>> k.add_env_keys(['VAR4']) # this has effect >>> k.add_cfg_keys(['cfg3']) >>> k.add_cmd_keys(['cmd3']) >>> k.get(['app']) ['app', 'cmd1', 'cmd2', 'cmd3', 'env1', 'env2', 'env3', 'env4', 'cfg1', 'cfg2', 'cfg3', 'DEFAULT'] Teardown: restore os module >>> os.getenv = _getenv """ def __init__(self): """Initialize Keys Instance""" self.cmd_keys = [] self.env_keys = [] self.cfg_keys = [] self.env_vars = [] def __repr__(self): """Return string representation of object""" return ','.join(self.get([])) def add_cmd_keys(self,keys): """Store keys from command line interface keys -- list of keys (typically from the command line) to store. May be a list of keys or a string with keys separated by commas. Use any value which evaluates False when no keys. """ self.cmd_keys.extend(split_keys(keys)) def add_env_keys(self,variables): """Store keys from invoking shell's environment variable variable -- (string) environment variable name from which to obtain keys to store """ variables = split_keys(variables) for variable in variables: # only add keys from shell environment variable if we haven't already if variable not in self.env_vars: # save key variable name so we can't add same keys twice self.env_vars.append(variable) # if shell environment variable has a option save it keys = os.getenv(variable,None) if keys is not None: self.env_keys.extend(split_keys(keys)) def add_cfg_keys(self,keys): """Store keys from user's configuration file. keys -- list of keys (from user's configuration file) to store. May be a list of keys or a string with keys separated by comma. Use any value which evaluates False when no keys. """ self.cfg_keys.extend(split_keys(keys)) def get(self,keys=None): """Return prioritized list of stored configuration keys keys -- list of differentiator keys. May be a list of keys or a string with keys separated by commas. Use any valid which evaluates False when no keys. """ keys = split_keys(keys) return (keys + self.cmd_keys + self.env_keys + self.cfg_keys + ['DEFAULT']) # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # O P T I O N V A L U E # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # class Value(object): """Used to store option settings in the blended option dictionary. Needed to be able to tie the option setting back to the configuration file for better error reporting and for round trip get/set/write capability.""" def __init__(self,value,parent,section_keys): """Initializes instance.""" self.value = value self.parent = parent self.section_keys = section_keys def set(self,value): """Sets option setting to 'value' argument passed in. By default configuration file parsers to do not support round trip. If they do they should subclass Value() and override this method """ self.value = value def get_roots(self): return ["File: %s" % self.parent.get_filename(), "Section: [%s]" % join_keys(self.section_keys,'.')] def __str__(self): """Returns string representation of the setting.""" return str(self.value) __repr__ = _repr def get(self): """Returns the option setting.""" return self.value # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # O P T I O N # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # class Option(object): """Options added to configuration parser are an instance of this class.""" def __init__(self,**kwargs): """Instance initializer. Arguments: kwargs -- dictionary with keys parser,name,type,default,help,check,keys, choices (see add_option of OptionContainer) """ self.__dict__.update(kwargs) if self.dest is None: self.dest = self.name if self.type not in self.conversions: TYPE_DOES_NOT_EXIST = "type '%s' is not valid" % self.type raise ConfigParserAppError(TYPE_DOES_NOT_EXIST) # help cross reference for options partnered with OptionParser self._help_xref = "" self.note = None def __str__(self): """Returns string representation of the option.""" return self.name __repr__ = _repr def add_note(self,note): """Adds 'note' argument text to configuration parser help text and to error messages associated with this option.""" self.parser.add_note(note) self.note = note # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # Get # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # def _get(self,keys): # Read any pending configuration files at the top level in order to # pick up user's default keys in those files. self.parser.parse_pending_cfg([]) keys = split_keys(keys) + self.keys keys = self.parser.keys.get(keys) # Get the option's dictionary from the configuration parser so that # any pending configuration files that are needed are read. option = self.parser.get_option_dict(self.name,keys) # When keys are given as an argument, we don't have the constraints # of an exact match like a section. Instead we use the default # key list (highest priority key first) to walk the option # dictionary. At each level of the dictionary we look for the # highest priority key and if it exists we move down a level # otherwise the remaining keys are checked in order of priority. def walk_option(option): if option.__class__ is dict: for key in keys: if key in option: v = walk_option(option[key]) if v.__class__ is not dict: return v if isinstance(option,Value): return option else: return NOTHING_FOUND return walk_option(option) def get(self,keys=[],errors=None): """Returns option value associated with 'keys' argument or options option value using 'keys' argument (in combination with other keys). keys -- name of keys to obtain option value from Note: If option is partnered with an optparse option and that option is present, the optparse option will take priority and be returned. """ warnings = [] option = NOTHING_FOUND def convert(value,option_help): # Do conversion to the type application specified value,warning = self.conversions[self.type](self,value) # Do final check using application check function if supplied if not warning and self.check is not None: value,warning = self.check(value) if warning: try: warnings.extend(option_help.get_roots()) except AttributeError: warnings.append(option_help) warnings.append(warning) return value # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # Priority 1: Get option from optparse partner (command line) # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # if self.dest in self.parser.optpar_option_partners: option = getattr(self.parser.optparser_options,self.dest,None) if option is None: option = NOTHING_FOUND else: option = convert(option,'command line option') # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # Priority 2: Get a default option # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # if option is NOTHING_FOUND and not warnings: option = self._get(keys) if option is not NOTHING_FOUND: value = option.get() option = convert(value,option) # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # Priority 3: Use default specified when adding the option # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # if option is NOTHING_FOUND and not warnings: if self.default is not NO_DEFAULT: option = self.default else: warnings.append('No valid default found.') keys = split_keys(keys) + self.keys keys = self.parser.keys.get(keys) warnings.append('keys=%s' % ','.join(keys)) if warnings: lines = ['Option: %s' % self.name] + warnings lines = '\n'.join(lines) + '\n' if errors is not None: errors.append(lines) else: # not coming back self.parser.write_errors([lines]) return option # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # Conversions # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # def convert_choice(self,value): if value in self.choices: return value,None else: choices = str(self.choices).strip('[]()') warning = "Invalid choice '%s', must be one of: %s" % (value,choices) return None,warning def convert_complex(self,value): try: return complex(value),None except ValueError: return None,"Cannot convert '%s' to a complex number" % (value) def convert_float(self,value): try: return float(value),None except ValueError: return None,"Cannot convert '%s' to a float" % (value) def convert_int(self,value): try: return int(value),None except ValueError: return None,"Cannot convert '%s' to an integer" % (value) def convert_long(self,value): try: return long(value),None except ValueError: return None,"Cannot convert '%s' to a long" % (value) def convert_string(self,value): try: return str(value),None except ValueError: return None,"Cannot convert '%s' to a string" % (value) def convert_nothing(self,value): return value,None conversions = { 'choice' : convert_choice, 'complex' : convert_complex, 'float' : convert_float, 'int' : convert_int, 'long' : convert_long, 'string' : convert_string, None : convert_nothing, } def set(self,value,cfgfile=None,keys=None): value = str(value) if cfgfile: if keys is not None: keys = split_keys(keys) else: keys = self.keys cfgfile.set_option(self.name,value,keys,self.help) else: keys = split_keys(keys) + self.keys keys = self.parser.keys.get(keys) option = self._get(keys) if option is NOTHING_FOUND: NOTHING_TO_SET = '\n'.join([ 'ERROR: No option found', 'option name: %s' % self.name, 'keys: %s' % keys]) raise ConfigParserUserError(NOTHING_TO_SET) else: option.set(value) cfgfile = option.parent return cfgfile # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # O P T I O N C O N T A I N E R B A S E C L A S S E S # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # class OptionContainer(object): OptionClass = Option def __init__(self,description): self.option_list = [] self.set_description(description) def set_description(self, description): self.description = description def get_description(self): return self.description def add_option(self,name,help=None,type=None,choices=None,dest=None,metavar=None,default=NO_DEFAULT,check=None,keys=None): """ name -- configuration file option name (used same as optparse) type -- choices similar to optparse (used same as optparse) default -- default value (used same as optparse) help -- help string (used same as optparse) dest -- option database attribute name (used same as optparse) check -- check function keys -- name of keys to obtain option from choices -- list of choices (used same as optparse) metavar -- FUTURE """ if dest is None: dest = name kwargs = { 'parser' : self.parser, 'name' : name, 'type' : type, 'help' : help, 'dest' : dest, 'check' : check, 'keys' : split_keys(keys), 'choices' : choices, 'metavar' : metavar, 'default' : default} option = self.OptionClass(**kwargs) self.parser.master_option_list.append(option) self.parser.master_option_dict[dest] = option self.option_list.append(option) return option # def format_option_help(self, formatter): if not self.option_list: return "" result = [] for option in self.option_list: if not option.help == SUPPRESS_HELP: result.append(formatter.format_option(option)) return "".join(result) def format_description(self, formatter): return formatter.format_description(self.get_description()) def format_help(self, formatter): result = [] if self.description: result.append(self.format_description(formatter)) if self.option_list: result.append(self.format_option_help(formatter)) return "\n".join(result) # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # O P T I O N G R O U P # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # class OptionGroup(OptionContainer): def __init__(self,parser,title,description): OptionContainer.__init__(self,description) self.parser = parser self.title = title # def set_title (self, title): self.title = title # -- Help-formatting methods --------------------------------------- def format_help (self, formatter): result = formatter.format_heading(self.title) formatter.indent() result += OptionContainer.format_help(self, formatter) formatter.dedent() return result # class ConfigFile(object): def __init__(self,cfgfile,content,keys,parent,parser): p,n = os.path.split(cfgfile) try: p = os.path.join(parent.path,p) except AttributeError: pass p = os.path.abspath(p) self.path = p self.filename = n self.content = content self.underkeys = keys self.parent = parent self.parser = parser self.parsed = False def get_filename(self): return os.path.join(self.path,self.filename) def __str__(self): return os.path.join(self.path,self.filename) __repr__ = _repr def get_as_str(self): content = self.content if content is None: cfgfile = os.path.join(self.path,self.filename) f = open(cfgfile) content = f.read() f.close() else: try: content = self.content.read() except AttributeError: pass return content def parse_if_unparsed(self): if not self.parsed: # The parent is important in that it is used error messages but more # importantly when getting ready to read a configuration script we # must set the current directory of the parent so relative path # specification of any configuration file works out. try: parent_path = self.parent.path except AttributeError: parent_path = os.getcwd() # Save so we can restore after we are done cwd = os.getcwd() # Make sure file is present cfgfile = os.path.join(self.path,self.filename) if not self.content and not os.path.exists(cfgfile): lines = ["File not found: '%s'" % cfgfile] # remember, top level configuration file parent is just the # current working directory when ConfigParser was instantiated # FUTURE parent info in here too FILE_NOT_FOUND = '\n'.join(lines) raise ConfigParserUserError(FILE_NOT_FOUND) # Change the current working directory to where the configuration # script is (to accomodate Python configuraton scripts so that it # can use os.getcwd() to determine its location). os.chdir(self.path) self.parse() # Restore os.chdir(cwd) # Mark it as done so it isn't parsed twice self.parsed = True class ConfigFilePy(ConfigFile): def parse(self): underkeys = self.underkeys parser = self.parser # Parsing can be easy! options = {} if self.content is None: cfgfile = os.path.join(self.path,self.filename) execfile(cfgfile,options) else: exec self.get_as_str() in options # Update the keys. "KEYS_VARIABLE" option used to specify the # environment variable that holds additional default keys, if # present get keys from the environment using it. If reading # configuration file that is being included under a key, don't # bother with its keys because it would get too confusing. if not underkeys: try: parser.keys.add_env_keys(options['KEYS_VARIABLE']) del options['KEYS_VARIABLE'] except KeyError: pass try: parser.keys.add_cfg_keys(options['KEYS']) del options['KEYS'] except KeyError: pass try: CONFIG_FILES = options['CONFIG_FILES'] del options['CONFIG_FILES'] except KeyError: CONFIG_FILES = None def merge_option(value,section_keys): if value.__class__ is dict: for key in value: merge_option(value[key],section_keys+[key]) else: valueobj = Value(value,self,section_keys) parser.merge_option(name,valueobj,underkeys+section_keys) # Merge all options that don't start with "_" into the options for name,value in options.items(): if not name.startswith('_'): merge_option(value,[]) # Process the "CONFIG_FILES" option used to merge in configuration # from other files. Users specify a comma separated (string) # listing of configuration file names or a dictionary of (and # optionally - of dictionaries of) file name strings. def add_files(value,under): if isinstance(value,dict): for k,v in value.iteritems(): add_files(v,under+[k]) else: for cf in split_paths(value): parser.add_file(cf,keys=under,parent=self) if CONFIG_FILES: add_files(CONFIG_FILES,underkeys) class ConfigFileIni(ConfigFile): def _read(self): try: self._already_read return except AttributeError: self._already_read = True underkeys = self.underkeys option_parser = self.parser marker_fmt = '{{{%s-(?P\d+)}}}' _self = self class BaseClass(object): def __init__(self,matchobj): self.text = matchobj.group('text') self.num = len(self.objects) self.objects.append(self) def restore(self): return self.text class Line(BaseClass): pass class Block(BaseClass): objects = [] find_re = re.compile('(?P.*?)',re.DOTALL) class Verbatim(BaseClass): objects = [] find_re = re.compile('(?P.*?)',re.DOTALL) class Comment(BaseClass): objects = [] find_re = re.compile('(?P[ \t]*[#;].*)') class OptionPair(Value): # OptionPair must subclass Value() because it is being submitted into the # parser options dictionary (all options in the dictionary must be a # subclass of Value). All things equal this would subclass BaseClass() # but it implements all that functionality anyways. objects = [] find_re = re.compile('(?P[ \t]*)(?P.+?)(?P\s*=\s*)(?P.*)') section = None def __init__(self,matchobj): section_keys = OptionPair.section.keys # get the name right name = matchobj.group('name') self.extended_name = name if '[' in name: name,keys = name.strip(']').split('[') section_keys = section_keys + split_keys(keys) self.name = name self.indent = matchobj.group('indent') self.sep = matchobj.group('sep') value = matchobj.group('value') self.valueplus = value try: value = restore(Comment,value) value = restore(Block,value) value = restore(Verbatim,value) self.linenum = re.findall(marker_fmt % 'Line',value)[0] except IndexError: self.linenum = 'new' self.num = len(self.objects) self.objects.append(self) self.section = OptionPair.section self.section.options[self.extended_name] = self # The following are needed for Value() functionality self.parent = _self self.section_keys = section_keys # self.value will be set later (can't now because value may contain # interpolations which cannot be expanded until all options are read # for this section def get_roots(self): return Value.get_roots(self) + ["Line: %s" % self.linenum] def set(self,value): Value.set(self,value) value = [value] def hit(matchobj): value.append(matchobj.group(0)) regexp = re.compile(marker_fmt % 'Comment') regexp.sub(hit,self.valueplus) self.valueplus = ''.join(value) def restore(self): return ''.join([self.extended_name,self.sep,self.valueplus]) def expand(self,levellist=[]): levellist = levellist + [self.name] if len(levellist) > 10: lines = self.get_roots() lines.append("Interpolation: %s" % ' << '.join(levellist)) lines.append("Maximum nesting level exceeded.") MAX_NESTING_LEVEL_EXCEEDED = '\n' + '\n'.join(lines) raise ConfigParserUserError(MAX_NESTING_LEVEL_EXCEEDED) try: return self.value except AttributeError: pass value = remove(Comment,self.valueplus) value = remove(Line,value) value = restore(Block,value.strip(' \t')) value = remove(Line,value) # @future [ABSPATH] value = value.replace('%(ABSPATH(','%_(ABSPATH(') regexp = re.compile('%\((?P.*?)\)s') def hit(matchobj): name = matchobj.group('name') try: return self.section.options[name].expand(levellist) except KeyError: try: return Section.default.options[name].expand(levellist) except KeyError: lines = self.get_roots() lines.append("Interpolation: %s" % ' << '.join(levellist+[name])) lines.append("'%s' not found in section or in [DEFAULT]." % name) INTERPOLATION_ERROR = '\n' + '\n'.join(lines) raise ConfigParserUserError(INTERPOLATION_ERROR) value = regexp.sub(hit,value) # @future [ABSPATH] regexp = re.compile(r'%_\(ABSPATH\((.*?)\)\)s') # def hit(matchobj): # return os.path.abspath(matchobj.group(1)) # value = regexp.sub(hit,value) self.value = remove(Line,restore(Verbatim,value)) return self.value class Section(BaseClass): objects = [] find_re = re.compile('(?P\n\[(?P.*?)\].*?(?=\n\[))',re.DOTALL) default = None def __init__(self,matchobj): self.options = {} name = matchobj.group('name') self.name = name keys = split_keys(name) if keys == ['DEFAULT']: keys = [] if not keys: Section.default = self self.keys = keys OptionPair.section = self self.text = collapse(OptionPair,matchobj.group(0)) self.num = len(self.objects) self.objects.append(self) def add_option(self,name,value,help): if help: helplines = textwrap.fill(help,77).split('\n') lines = ['# %s' % line for line in helplines] else: lines = [] lines.append('%s = %s' % (name,value)) OptionPair.section = self block = collapse(Comment,'\n'+'\n'.join(lines)) self.text += collapse(OptionPair,block) option = OptionPair.objects[-1] underkeys = _self.underkeys + option.section_keys if not underkeys: underkeys = ['DEFAULT'] # submit new value to parser _self.parser.merge_option(name,option,underkeys) return option def collapse(SubClass,text): marker_fmt = '{{{%s-%%d}}}' % (SubClass.__name__) def hit(matchobj): return marker_fmt % SubClass(matchobj).num return SubClass.find_re.sub(hit,text) def remove(SubClass,text): return re.compile(marker_fmt % SubClass.__name__).sub('',text) def restore(SubClass,text): def hit(matchobj): return SubClass.objects[int(matchobj.group('id'))].restore() return re.compile(marker_fmt % SubClass.__name__).sub(hit,text) text = self.get_as_str() lines = [] i = 1 for line in text.split('\n'): lines.append(line + ('{{{Line-%d}}}' % i)) i += 1 text = '\n'.join(lines) text = '\n[DEFAULT]\n#_START_MARKER_\n%s\n[' % text text = collapse(Block,text) text = collapse(Verbatim,text) text = collapse(Comment,text) text = collapse(Section,text) self._OptionPair = OptionPair self._Line = Line self._Comment = Comment self._Block = Block self._Verbatim = Verbatim self._Section = Section self._collapse = collapse self._restore = restore self._remove = remove self.text = text def parse(self): self._read() for option in self._OptionPair.objects: name = option.name underkeys = self.underkeys + option.section_keys value = option.expand() if name == '': for cf in split_paths(value): self.parser.add_file(cf,keys=underkeys,parent=self) continue if not underkeys: if name == '': self.parser.keys.add_cfg_keys(split_keys(value)) continue if name == '': self.parser.keys.add_env_keys(value) continue underkeys = ['DEFAULT'] # call expand method to get .value attribute set self.parser.merge_option(name,option,underkeys) def set_option(self,name,value,keys=None,help=None): self._read() value = str(value) keys = split_keys(keys) found = False for option in self._OptionPair.objects: if name==option.name and keys == option.section_keys: option.set(value) found = True if not found: Section = self._Section for section in Section.objects: if keys == section.keys: found = True section.add_option(name,value,help) if not found: block = '\n\n[%s]\n\n[\n' % join_keys(keys,'.') self.text = '%s%s' % (self.text[:-2],self._collapse(Section,block)) section = self._Section.objects[-1] section.add_option(name,value,help) def write(self,file): self._read() restore = self._restore content = self.text content = restore(self._Section,content) content = restore(self._OptionPair,content) + '\n' content = restore(self._Comment,content) content = restore(self._Block,content) content = restore(self._Verbatim,content) content = self._remove(self._Line,content) content = content.split('\n#_START_MARKER_\n')[1][:-3] try: file.write(content) except AttributeError: f = open(file,'w') f.write(content) f.close() # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # C O N F I G U R A T I O N P A R S E R # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # class ConfigParser(OptionContainer): KeysClass = Keys OptionGroupClass = OptionGroup # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # Initializer # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # def __init__(self,description=None,allow_py=False,formatter=None,exception=False): """ description -- Introductory text placed above configuration option help text. allow_py -- Set to True to allow Python based configuraton files to be executed. Defaults to False. Enabling this feature poses a potential security hole for your application. formatter -- Controls configuration option help text style. Set to either the IndentedHelpFormatter or TitledHelpFormatter classes found in cfgparse module (or a subclass of either). exception -- set to True to raise ConfigParserUserError exception when configuration error occurs (due to user error). Omitting or setting to False will write error message to sys.stderr. Set to an exception class to raise that exception when a user configuration file error occurs. """ OptionContainer.__init__(self,description) self.exception = exception # Needed because shares same base class as an option group # (option group constructor initializes parser using an arg). self.parser = self self.option_dicts = {} self.option_groups = [] self.optpar_option_partners = {} self.master_option_list = [] self.master_option_dict = {} if formatter is None: formatter = IndentedHelpFormatter() self.formatter = formatter self.formatter.set_parser(self) self.notes = [] # Set up database of option selection keys self.keys = self.KeysClass() # Create database to store information on those configuration files # to be read later (configuration files under keys are not read until # all of the keys in which it is under are active. self._pending = [] # Since it introduces a security risk, only allow Python based # configuration files if application explicitly sets this True. self.allow_python_cfg = allow_py self.optparse_dests = {} def add_optparse_keys_option(self,option_group,switches=('-k','--keys'),dest='cfgparse_keys',help="Comma separated list of configuration keys."): """Adds configuration file keys list option to optparse object.""" self.optparse_dests['keys'] = dest option_group.add_option(dest=dest,metavar='LIST',help=help,*switches) def add_optparse_files_option(self,option_group,switches=('--cfgfiles',),dest='cfgparse_files',help="Comma separated list of configuration files."): """Adds configuration file list option to optparse object.""" self.optparse_dests['files'] = dest option_group.add_option(dest=dest,metavar='LIST',help=help,*switches) def add_optparse_help_option(self,option_group,switches=('--cfghelp',),dest='cfgparse_help',help="Show configuration file help and exit."): """Adds configuration file help option to optparse object.""" self.optparse_dests['help'] = dest option_group.add_option(dest=dest,action='store_true',help=help,*switches) def add_env_file(self,var,keys=[]): """Adds configuration file specified by environment variable setting. Arguments: var -- name of environment variable holding configuration file name keys -- section key list to place configuration file options settings under """ # Add configuration files specified by an environment variable # if application script specified it. (Don't read right away, # rather hold them in pending database until they are needed # because adding options causes option_dicts to be set with a # default for the option destination. f = os.getenv(var,None) if f: f = self.add_file(cfgfile=f,keys=keys) else: f = None return f def get_option(self,dest): """Returns option object previously added. Arguments: dest -- destination attribute name of option """ opt = self.master_option_dict.get(dest,None) if opt is None: OPTION_NOT_FOUND = '\n'.join([ 'ERROR: No option found', 'option dest: %s' % dest]) raise ConfigParserAppError(OPTION_NOT_FOUND) return opt # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # Adding Option Groups (adding options handled by base class) # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # def add_option_group(self,title,description=None): option_group = self.OptionGroupClass(self,title,description) self.option_groups.append(option_group) return option_group # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # Pending Configuration # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # def get_option_dict(self,name,keys): self.parse_pending_cfg(keys) return self.option_dicts.get(name,NOTHING_FOUND) def parse_pending_cfg(self,keys=[],read_all=False): """ read_all -- Set to True to read all configuration files up front. Defaults to reading "on the fly" as the configuration files are needed.""" # Read any pending configuration files that could possibly effect the # setting about to be retrieved. Note, it was decided that if the # pending configuration file's under keys are all in the key list # computed above it will be installed right away. The other # alternative is to try retrieving the setting with the highest # priority key alone (first reading any pending configuration files # that are under that key), then if that fails try retrieving the # setting with the top two highest priority keys (again first reading # any pending configuration files that are under either or both of the # keys), and repeating this process for each key in the list until a # setting is found. This would have the benefit of only reading # pending configuration files when it is absolutely necessary but at # cost of performance and more difficult to explain how it works. keys = split_keys(keys) d = [] while self._pending: underkeys,cfgfileobj = self._pending.pop(0) underkeys = split_keys(underkeys) parse_it = read_all if not parse_it: for key in underkeys: if key not in keys: d.append((underkeys,cfgfileobj)) break else: parse_it = True if parse_it: cfgfileobj.parse_if_unparsed() self._pending = d # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # FileClasses = {'py' : ConfigFilePy, 'ini': ConfigFileIni, 'default' : ConfigFileIni} def add_file(self,cfgfile=None,content=None,type=None,keys=[],parent=None): """ Adds configuration file to parser. Note, file is not read until parse or get_option method is called (and even then it may not be read if any keys in the keys list are not in the keys being used to parse). cfgfile -- either the filename or a file stream, defaults to None (see table below) content -- either file contents string or file stream, defaults to None (see table below) type -- set to either 'py', 'ini', or None (default) to control file parser used. When set to None, filename extension is used to determine parser to use. 'py' interprets the file as Python code. Otherwise the 'ini' style parser is used. keys -- key list that all options in the configuration file will be placed under when the file is read. parent -- Not intended to be used by the public The following table summarizes the legal combinations of the cfgfile and arguments and resulting file name and contents utilized. cfgfile content result (when configuration is parsed) -------- ------- ----------------------------------------------------------- filename None file is opened and contents read stream None stream contents are read, filename is stream name attribute filename stream stream contents are read, filename is cfgfile argument filename string both filename and content are used as is """ if isinstance(cfgfile,str): n = cfgfile c = content elif hasattr(cfgfile,'name'): n = cfgfile.name c = cfgfile elif isinstance(content,str): n = 'heredoc' c = content elif hasattr(content,'read'): n = 'stream' c = content else: ARGUMENT_CONFICT = "Illegal combination of 'cfgfile' and 'content' arguments" raise ConfigParserAppError(ARGUMENT_CONFICT) uk = split_keys(keys) if type is None: # calculate type (lower case extension) type = os.path.splitext(n)[1][1:].lower() if type == 'py' and not self.allow_python_cfg: return None FileClass = self.FileClasses.get(type) if FileClass is None: FileClass = self.FileClasses['default'] fileobj = FileClass(cfgfile=n,content=c,keys=uk,parent=parent,parser=self) self._pending.append((uk,fileobj)) return fileobj # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # Read Configuration File Utilities # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # def merge_option(self,name,value,keys): """ Merge value (under keys) into options dictionary. name -- option name value -- value to assign option (may be a dictionary or a option) keys -- list of keys to place value in options dict under """ # Add the option name to the key list so we can start walking at # the top level of the options dictionary. keys = [name] + keys option_dicts = self.option_dicts # Move through the options dictionary using the keys we are # supposed to place the value under creating dictionaries and keys # that aren't already present. for key in keys[:-1]: if key in option_dicts: if option_dicts[key].__class__ is not dict: option_dicts[key] = dict(DEFAULT=option_dicts[key]) else: option_dicts[key]={} option_dicts = option_dicts[key] option_dicts[keys[-1]] = value # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # Option Parsing (not configuration file parsing) # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # def write_errors(self,errors): CFGPARSE_USER_ERROR = '\n' + '\n'.join(errors) if not self.exception: sys.stderr.write("ERROR: Configuration File Parser\n") sys.stderr.write(CFGPARSE_USER_ERROR) self.system_exit() else: if self.exception is True: UserError = ConfigParserUserError else: UserError = self.exception raise UserError(CFGPARSE_USER_ERROR) # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # Option Parsing # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # def parse(self,optparser=None,args=None,read_all=False): """Partners the option parser and the configuration parser together read_all -- Set to True to read all configuration files up front. Defaults to reading "on the fly" as the configuration files are needed. """ if optparser: # Marry up type, help, choices attributes between option parser and # configuration parser options. self.marry_options(optparser) # Parse command line arguments options, args = optparser.parse_args(args) self.optparser_options = options # generate help if requested from the command line help_dest = self.optparse_dests.get('help') if help_dest and getattr(options,help_dest): self.print_help() self.system_exit() # add command line keys keys_dest = self.optparse_dests.get('keys') if keys_dest: self.keys.add_cmd_keys(getattr(options,keys_dest)) # add command line configuration files (must hold it as other configuration # files may be pending that should be read first) files_dest = self.optparse_dests.get('files') if files_dest: cfgfiles = getattr(options,files_dest) for cf in split_paths(cfgfiles): self.add_file(cfgfile=cf,keys=[]) else: class ConfigOptions(object): pass options = ConfigOptions() self.parse_pending_cfg([],read_all) # Go through each option in the configuration options and add them # to options object. errors = [] for option in self.master_option_list: setattr(options,option.dest,option.get(errors=errors)) if errors: self.write_errors(errors) if optparser: return options, args else: return options # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # def marry_options(self,optparser): # create mapping: dest -> [optpar options] optpar_lookup = {} for option in optparser.option_list: if option.dest: optpar_lookup.setdefault(option.dest,[]).append(option) for group in optparser.option_groups: for option in group.option_list: if option.dest: optpar_lookup.setdefault(option.dest,[]).append(option) self.optpar_option_partners = optpar_lookup # we are guarenteed no duplicate destinations with a configuration parser for cfgpar_option in self.master_option_list: optpar_options = optpar_lookup.get(cfgpar_option.dest,[]) for optpar_option in optpar_options: for attrname in ['metavar','type','choices','help']: cfgpar_attr = getattr(cfgpar_option,attrname) optpar_attr = getattr(optpar_option,attrname) if cfgpar_attr is None: cfgpar_attr = optpar_attr setattr(cfgpar_option,attrname,cfgpar_attr) if cfgpar_option.help == SUPPRESS_HELP: continue try: # remove anything we've added previously optpar_option.help = optpar_option.help.replace(optpar_option._cfgparse_help,'') except AttributeError: # must not have modified it previously pass help = " See also '%s' option in configuration file help." % (cfgpar_option.name) if not optpar_option.help: help = help.strip() optpar_option.help = '' optpar_option.help = optpar_option.help + help optpar_option._cfgparse_help = help if cfgpar_option.help == SUPPRESS_HELP: continue if cfgpar_option.help is None: cfgpar_option.help = '' try: # remove anything we've added previously cfgpar_option.help = cfgpar_option.help.replace(cfgpar_option._help_xref,'') except AttributeError: # must not have modified it previously pass switches = '/'.join([str(o) for o in optpar_options]) if switches: help = " See also '%s' command line switch." % (switches) if not cfgpar_option.help: help = help.strip() cfgpar_option.help = cfgpar_option.help + help cfgpar_option._help_xref = help # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # def marry_attribute(self,attrname,cfgpar_option,optpar_option): cfgpar_attr = getattr(cfgpar_option,attrname) optpar_attr = getattr(optpar_option,attrname) if cfgpar_attr is None: cfgpar_attr = optpar_attr setattr(cfgpar_option,attrname,cfgpar_attr) # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # Help # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # def format_option_help(self, formatter=None): if formatter is None: formatter = self.formatter formatter.store_option_strings(self) result = [] result.append(formatter.format_heading(_("Configuration file options"))) formatter.indent() if self.option_list: result.append(OptionContainer.format_option_help(self, formatter)) result.append("\n") for group in self.option_groups: result.append(group.format_help(formatter)) result.append("\n") formatter.dedent() # Drop the last "\n", or the header if no options or option groups: return "".join(result[:-1]) def format_help(self, formatter=None): if formatter is None: formatter = self.formatter result = [] if self.description: result.append(self.format_description(formatter) + "\n") result.append(self.format_option_help(formatter)) return "".join(result) def print_help(self, file=None): """print_help(file : file = stdout) Print an extended help message, listing all options and any help text provided with them, to 'file' (default stdout). """ if file is None: file = sys.stdout file.write(self.format_help()) if self.notes: file.write('\nNotes:\n%s\n'%'\n'.join(self.notes)) # def add_note(self,note): self.notes.append(note) def system_exit(self): sys.exit() def _test(): import doctest doctest.testmod() if __name__ == "__main__": _test()