Source code for tox.config

import argparse
import os
import random
from fnmatch import fnmatchcase
import sys
import re
import shlex
import string
import pkg_resources
import itertools
import pluggy

import tox.interpreters
from tox import hookspecs
from tox._verlib import NormalizedVersion

import py

import tox

iswin32 = sys.platform == "win32"

default_factors = {'jython': 'jython', 'pypy': 'pypy', 'pypy3': 'pypy3',
                   'py': sys.executable}
for version in '26,27,32,33,34,35,36'.split(','):
    default_factors['py' + version] = 'python%s.%s' % tuple(version)

hookimpl = pluggy.HookimplMarker("tox")

_dummy = object()

def get_plugin_manager(plugins=()):
    # initialize plugin manager
    import tox.venv
    pm = pluggy.PluginManager("tox")
    for plugin in plugins:
    return pm

[docs]class Parser: """ command line and ini-parser control object. """ def __init__(self): self.argparser = argparse.ArgumentParser( description="tox options", add_help=False) self._testenv_attr = []
[docs] def add_argument(self, *args, **kwargs): """ add argument to command line parser. This takes the same arguments that ``argparse.ArgumentParser.add_argument``. """ return self.argparser.add_argument(*args, **kwargs)
[docs] def add_testenv_attribute(self, name, type, help, default=None, postprocess=None): """ add an ini-file variable for "testenv" section. Types are specified as strings like "bool", "line-list", "string", "argv", "path", "argvlist". The ``postprocess`` function will be called for each testenv like ``postprocess(testenv_config=testenv_config, value=value)`` where ``value`` is the value as read from the ini (or the default value) and ``testenv_config`` is a :py:class:`tox.config.TestenvConfig` instance which will receive all ini-variables as object attributes. Any postprocess function must return a value which will then be set as the final value in the testenv section. """ self._testenv_attr.append(VenvAttribute(name, type, default, help, postprocess))
[docs] def add_testenv_attribute_obj(self, obj): """ add an ini-file variable as an object. This works as the ``add_testenv_attribute`` function but expects "name", "type", "help", and "postprocess" attributes on the object. """ assert hasattr(obj, "name") assert hasattr(obj, "type") assert hasattr(obj, "help") assert hasattr(obj, "postprocess") self._testenv_attr.append(obj)
def _parse_args(self, args): return self.argparser.parse_args(args) def _format_help(self): return self.argparser.format_help()
class VenvAttribute: def __init__(self, name, type, default, help, postprocess): = name self.type = type self.default = default = help self.postprocess = postprocess class DepOption: name = "deps" type = "line-list" help = "each line specifies a dependency in pip/setuptools format." default = () def postprocess(self, testenv_config, value): deps = [] config = testenv_config.config for depline in value: m = re.match(r":(\w+):\s*(\S+)", depline) if m: iname, name = m.groups() ixserver = config.indexserver[iname] else: name = depline.strip() ixserver = None name = self._replace_forced_dep(name, config) deps.append(DepConfig(name, ixserver)) return deps def _replace_forced_dep(self, name, config): """ Override the given dependency config name taking --force-dep-version option into account. :param name: dep config, for example ["pkg==1.0", "other==2.0"]. :param config: Config instance :return: the new dependency that should be used for virtual environments """ if not config.option.force_dep: return name for forced_dep in config.option.force_dep: if self._is_same_dep(forced_dep, name): return forced_dep return name @classmethod def _is_same_dep(cls, dep1, dep2): """ Returns True if both dependency definitions refer to the same package, even if versions differ. """ dep1_name = pkg_resources.Requirement.parse(dep1).project_name try: dep2_name = pkg_resources.Requirement.parse(dep2).project_name except pkg_resources.RequirementParseError: # we couldn't parse a version, probably a URL return False return dep1_name == dep2_name class PosargsOption: name = "args_are_paths" type = "bool" default = True help = "treat positional args in commands as paths" def postprocess(self, testenv_config, value): config = testenv_config.config args = config.option.args if args: if value: args = [] for arg in config.option.args: if arg: origpath = config.invocationcwd.join(arg, abs=True) if origpath.check(): arg = testenv_config.changedir.bestrelpath(origpath) args.append(arg) testenv_config._reader.addsubstitutions(args) return value class InstallcmdOption: name = "install_command" type = "argv" default = "pip install {opts} {packages}" help = "install command for dependencies and package under test." def postprocess(self, testenv_config, value): if '{packages}' not in value: raise tox.exception.ConfigError( "'install_command' must contain '{packages}' substitution") return value def parseconfig(args=None, plugins=()): """ :param list[str] args: Optional list of arguments. :type pkg: str :rtype: :class:`Config` :raise SystemExit: toxinit file is not found """ pm = get_plugin_manager(plugins) if args is None: args = sys.argv[1:] # prepare command line options parser = Parser() pm.hook.tox_addoption(parser=parser) # parse command line options option = parser._parse_args(args) interpreters = tox.interpreters.Interpreters(hook=pm.hook) config = Config(pluginmanager=pm, option=option, interpreters=interpreters) config._parser = parser config._testenv_attr = parser._testenv_attr # parse ini file basename = config.option.configfile if os.path.isabs(basename): inipath = py.path.local(basename) else: for path in py.path.local().parts(reverse=True): inipath = path.join(basename) if inipath.check(): break else: inipath = py.path.local().join('setup.cfg') if not inipath.check(): feedback("toxini file %r not found" % (basename), sysexit=True) try: parseini(config, inipath) except tox.exception.InterpreterNotFound: exn = sys.exc_info()[1] # Use stdout to match test expectations py.builtin.print_("ERROR: " + str(exn)) # post process config object pm.hook.tox_configure(config=config) return config def feedback(msg, sysexit=False): py.builtin.print_("ERROR: " + msg, file=sys.stderr) if sysexit: raise SystemExit(1) class VersionAction(argparse.Action): def __call__(self, argparser, *args, **kwargs): version = tox.__version__ py.builtin.print_("%s imported from %s" % (version, tox.__file__)) raise SystemExit(0) class CountAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): if hasattr(namespace, self.dest): setattr(namespace, self.dest, int(getattr(namespace, self.dest)) + 1) else: setattr(namespace, self.dest, 0) class SetenvDict: def __init__(self, dict, reader): self.reader = reader self.definitions = dict self.resolved = {} self._lookupstack = [] def __contains__(self, name): return name in self.definitions def get(self, name, default=None): try: return self.resolved[name] except KeyError: try: if name in self._lookupstack: raise KeyError(name) val = self.definitions[name] except KeyError: return os.environ.get(name, default) self._lookupstack.append(name) try: self.resolved[name] = res = self.reader._replace(val) finally: self._lookupstack.pop() return res def __getitem__(self, name): x = self.get(name, _dummy) if x is _dummy: raise KeyError(name) return x def keys(self): return self.definitions.keys() def __setitem__(self, name, value): self.definitions[name] = value self.resolved[name] = value @hookimpl def tox_addoption(parser): # formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("--version", nargs=0, action=VersionAction, dest="version", help="report version information to stdout.") parser.add_argument("-h", "--help", action="store_true", dest="help", help="show help about options") parser.add_argument("--help-ini", "--hi", action="store_true", dest="helpini", help="show help about ini-names") parser.add_argument("-v", nargs=0, action=CountAction, default=0, dest="verbosity", help="increase verbosity of reporting output.") parser.add_argument("--showconfig", action="store_true", help="show configuration information for all environments. ") parser.add_argument("-l", "--listenvs", action="store_true", dest="listenvs", help="show list of test environments") parser.add_argument("-c", action="store", default="tox.ini", dest="configfile", help="use the specified config file name.") parser.add_argument("-e", action="append", dest="env", metavar="envlist", help="work against specified environments (ALL selects all).") parser.add_argument("--notest", action="store_true", dest="notest", help="skip invoking test commands.") parser.add_argument("--sdistonly", action="store_true", dest="sdistonly", help="only perform the sdist packaging activity.") parser.add_argument("--installpkg", action="store", default=None, metavar="PATH", help="use specified package for installation into venv, instead of " "creating an sdist.") parser.add_argument("--develop", action="store_true", dest="develop", help="install package in the venv using ' develop' via " "'pip -e .'") parser.add_argument('-i', action="append", dest="indexurl", metavar="URL", help="set indexserver url (if URL is of form name=url set the " "url for the 'name' indexserver, specifically)") parser.add_argument("--pre", action="store_true", dest="pre", help="install pre-releases and development versions of dependencies. " "This will pass the --pre option to install_command " "(pip by default).") parser.add_argument("-r", "--recreate", action="store_true", dest="recreate", help="force recreation of virtual environments") parser.add_argument("--result-json", action="store", dest="resultjson", metavar="PATH", help="write a json file with detailed information " "about all commands and results involved.") # We choose 1 to 4294967295 because it is the range of PYTHONHASHSEED. parser.add_argument("--hashseed", action="store", metavar="SEED", default=None, help="set PYTHONHASHSEED to SEED before running commands. " "Defaults to a random integer in the range [1, 4294967295] " "([1, 1024] on Windows). " "Passing 'noset' suppresses this behavior.") parser.add_argument("--force-dep", action="append", metavar="REQ", default=None, help="Forces a certain version of one of the dependencies " "when configuring the virtual environment. REQ Examples " "'pytest<2.7' or 'django>=1.6'.") parser.add_argument("--sitepackages", action="store_true", help="override sitepackages setting to True in all envs") parser.add_argument("--skip-missing-interpreters", action="store_true", help="don't fail tests for missing interpreters") parser.add_argument("--workdir", action="store", dest="workdir", metavar="PATH", default=None, help="tox working directory") parser.add_argument("args", nargs="*", help="additional arguments available to command positional substitution") parser.add_testenv_attribute( name="envdir", type="path", default="{toxworkdir}/{envname}", help="set venv directory -- be very careful when changing this as tox " "will remove this directory when recreating an environment") # add various core venv interpreter attributes def setenv(testenv_config, value): setenv = value config = testenv_config.config if "PYTHONHASHSEED" not in setenv and config.hashseed is not None: setenv['PYTHONHASHSEED'] = config.hashseed return setenv parser.add_testenv_attribute( name="setenv", type="dict_setenv", postprocess=setenv, help="list of X=Y lines with environment variable settings") def basepython_default(testenv_config, value): if value is None: for f in testenv_config.factors: if f in default_factors: return default_factors[f] return sys.executable return str(value) parser.add_testenv_attribute( name="basepython", type="string", default=None, postprocess=basepython_default, help="executable name or path of interpreter used to create a " "virtual test environment.") parser.add_testenv_attribute( name="envtmpdir", type="path", default="{envdir}/tmp", help="venv temporary directory") parser.add_testenv_attribute( name="envlogdir", type="path", default="{envdir}/log", help="venv log directory") def downloadcache(testenv_config, value): if value: # env var, if present, takes precedence downloadcache = os.environ.get("PIP_DOWNLOAD_CACHE", value) return py.path.local(downloadcache) parser.add_testenv_attribute( name="downloadcache", type="string", default=None, postprocess=downloadcache, help="(deprecated) set PIP_DOWNLOAD_CACHE.") parser.add_testenv_attribute( name="changedir", type="path", default="{toxinidir}", help="directory to change to when running commands") parser.add_testenv_attribute_obj(PosargsOption()) parser.add_testenv_attribute( name="skip_install", type="bool", default=False, help="Do not install the current package. This can be used when " "you need the virtualenv management but do not want to install " "the current package") parser.add_testenv_attribute( name="ignore_errors", type="bool", default=False, help="if set to True all commands will be executed irrespective of their " "result error status.") def recreate(testenv_config, value): if testenv_config.config.option.recreate: return True return value parser.add_testenv_attribute( name="recreate", type="bool", default=False, postprocess=recreate, help="always recreate this test environment.") def passenv(testenv_config, value): # Flatten the list to deal with space-separated values. value = list( itertools.chain.from_iterable( [x.split(' ') for x in value])) passenv = set(["PATH", "PIP_INDEX_URL", "LANG", "LD_LIBRARY_PATH"]) # read in global passenv settings p = os.environ.get("TOX_TESTENV_PASSENV", None) if p is not None: passenv.update(x for x in p.split() if x) # we ensure that tmp directory settings are passed on # we could also set it to the per-venv "envtmpdir" # but this leads to very long paths when run with jenkins # so we just pass it on by default for now. if sys.platform == "win32": passenv.add("SYSTEMDRIVE") # needed for pip6 passenv.add("SYSTEMROOT") # needed for python's crypto module passenv.add("PATHEXT") # needed for discovering executables passenv.add("TEMP") passenv.add("TMP") else: passenv.add("TMPDIR") for spec in value: for name in os.environ: if fnmatchcase(name.upper(), spec.upper()): passenv.add(name) return passenv parser.add_testenv_attribute( name="passenv", type="line-list", postprocess=passenv, help="environment variables needed during executing test commands " "(taken from invocation environment). Note that tox always " "passes through some basic environment variables which are " "needed for basic functioning of the Python system. " "See --showconfig for the eventual passenv setting.") parser.add_testenv_attribute( name="whitelist_externals", type="line-list", help="each lines specifies a path or basename for which tox will not warn " "about it coming from outside the test environment.") parser.add_testenv_attribute( name="platform", type="string", default=".*", help="regular expression which must match against ``sys.platform``. " "otherwise testenv will be skipped.") def sitepackages(testenv_config, value): return testenv_config.config.option.sitepackages or value parser.add_testenv_attribute( name="sitepackages", type="bool", default=False, postprocess=sitepackages, help="Set to ``True`` if you want to create virtual environments that also " "have access to globally installed packages.") def pip_pre(testenv_config, value): return testenv_config.config.option.pre or value parser.add_testenv_attribute( name="pip_pre", type="bool", default=False, postprocess=pip_pre, help="If ``True``, adds ``--pre`` to the ``opts`` passed to " "the install command. ") def develop(testenv_config, value): option = testenv_config.config.option return not option.installpkg and (value or option.develop) parser.add_testenv_attribute( name="usedevelop", type="bool", postprocess=develop, default=False, help="install package in develop/editable mode") parser.add_testenv_attribute_obj(InstallcmdOption()) parser.add_testenv_attribute( name="list_dependencies_command", type="argv", default="pip freeze", help="list dependencies for a virtual environment") parser.add_testenv_attribute_obj(DepOption()) parser.add_testenv_attribute( name="commands", type="argvlist", default="", help="each line specifies a test command and can use substitution.") parser.add_testenv_attribute( "ignore_outcome", type="bool", default=False, help="if set to True a failing result of this testenv will not make " "tox fail, only a warning will be produced") parser.add_testenv_attribute( "extras", type="line-list", help="list of extras to install with the source distribution or " "develop install")
[docs]class Config(object): """ Global Tox config object. """ def __init__(self, pluginmanager, option, interpreters): #: dictionary containing envname to envconfig mappings self.envconfigs = {} self.invocationcwd = py.path.local() self.interpreters = interpreters self.pluginmanager = pluginmanager #: option namespace containing all parsed command line options self.option = option @property def homedir(self): homedir = get_homedir() if homedir is None: homedir = self.toxinidir # XXX good idea? return homedir
[docs]class TestenvConfig: """ Testenv Configuration object. In addition to some core attributes/properties this config object holds all per-testenv ini attributes as attributes, see "tox --help-ini" for an overview. """ def __init__(self, envname, config, factors, reader): #: test environment name self.envname = envname #: global tox config object self.config = config #: set of factors self.factors = factors self._reader = reader
[docs] def get_envbindir(self): """ path to directory where scripts/binaries reside. """ if (sys.platform == "win32" and "jython" not in self.basepython and "pypy" not in self.basepython): return self.envdir.join("Scripts") else: return self.envdir.join("bin")
@property def envbindir(self): return self.get_envbindir() @property def envpython(self): """ path to python executable. """ return self.get_envpython()
[docs] def get_envpython(self): """ path to python/jython executable. """ if "jython" in str(self.basepython): name = "jython" else: name = "python" return self.envbindir.join(name)
[docs] def get_envsitepackagesdir(self): """ return sitepackagesdir of the virtualenv environment. (only available during execution, not parsing) """ x = self.config.interpreters.get_sitepackagesdir( info=self.python_info, envdir=self.envdir) return x
@property def python_info(self): """ return sitepackagesdir of the virtualenv environment. """ return self.config.interpreters.get_info(envconfig=self) def getsupportedinterpreter(self): if sys.platform == "win32" and self.basepython and \ "jython" in self.basepython: raise tox.exception.UnsupportedInterpreter( "Jython/Windows does not support installing scripts") info = self.config.interpreters.get_info(envconfig=self) if not info.executable: raise tox.exception.InterpreterNotFound(self.basepython) if not info.version_info: raise tox.exception.InvocationError( 'Failed to get version_info for %s: %s' % (, info.err)) if info.version_info < (2, 6): raise tox.exception.UnsupportedInterpreter( "python2.5 is not supported anymore, sorry") return info.executable
testenvprefix = "testenv:" def get_homedir(): try: return py.path.local._gethomedir() except Exception: return None def make_hashseed(): max_seed = 4294967295 if sys.platform == 'win32': max_seed = 1024 return str(random.randint(1, max_seed)) class parseini: def __init__(self, config, inipath): config.toxinipath = inipath config.toxinidir = config.toxinipath.dirpath() self._cfg = py.iniconfig.IniConfig(config.toxinipath) config._cfg = self._cfg self.config = config if inipath.basename == 'setup.cfg': prefix = 'tox' else: prefix = None ctxname = getcontextname() if ctxname == "jenkins": reader = SectionReader("tox:jenkins", self._cfg, prefix=prefix, fallbacksections=['tox']) distshare_default = "{toxworkdir}/distshare" elif not ctxname: reader = SectionReader("tox", self._cfg, prefix=prefix) distshare_default = "{homedir}/.tox/distshare" else: raise ValueError("invalid context") if config.option.hashseed is None: hashseed = make_hashseed() elif config.option.hashseed == 'noset': hashseed = None else: hashseed = config.option.hashseed config.hashseed = hashseed reader.addsubstitutions(toxinidir=config.toxinidir, homedir=config.homedir) # As older versions of tox may have bugs or incompatabilities that # prevent parsing of tox.ini this must be the first thing checked. config.minversion = reader.getstring("minversion", None) if config.minversion: minversion = NormalizedVersion(self.config.minversion) toxversion = NormalizedVersion(tox.__version__) if toxversion < minversion: raise tox.exception.MinVersionError( "tox version is %s, required is at least %s" % ( toxversion, minversion)) if config.option.workdir is None: config.toxworkdir = reader.getpath("toxworkdir", "{toxinidir}/.tox") else: config.toxworkdir = config.toxinidir.join(config.option.workdir, abs=True) if not config.option.skip_missing_interpreters: config.option.skip_missing_interpreters = \ reader.getbool("skip_missing_interpreters", False) # determine indexserver dictionary config.indexserver = {'default': IndexServerConfig('default')} prefix = "indexserver" for line in reader.getlist(prefix): name, url = map(lambda x: x.strip(), line.split("=", 1)) config.indexserver[name] = IndexServerConfig(name, url) override = False if config.option.indexurl: for urldef in config.option.indexurl: m = re.match(r"\W*(\w+)=(\S+)", urldef) if m is None: url = urldef name = "default" else: name, url = m.groups() if not url: url = None if name != "ALL": config.indexserver[name].url = url else: override = url # let ALL override all existing entries if override: for name in config.indexserver: config.indexserver[name] = IndexServerConfig(name, override) reader.addsubstitutions(toxworkdir=config.toxworkdir) config.distdir = reader.getpath("distdir", "{toxworkdir}/dist") reader.addsubstitutions(distdir=config.distdir) config.distshare = reader.getpath("distshare", distshare_default) reader.addsubstitutions(distshare=config.distshare) config.sdistsrc = reader.getpath("sdistsrc", None) config.setupdir = reader.getpath("setupdir", "{toxinidir}") config.logdir = config.toxworkdir.join("log") config.envlist, all_envs = self._getenvdata(reader) # factors used in config or predefined known_factors = self._list_section_factors("testenv") known_factors.update(default_factors) known_factors.add("python") # factors stated in config envlist stated_envlist = reader.getstring("envlist", replace=False) if stated_envlist: for env in _split_env(stated_envlist): known_factors.update(env.split('-')) # configure testenvs for name in all_envs: section = testenvprefix + name factors = set(name.split('-')) if section in self._cfg or factors <= known_factors: config.envconfigs[name] = \ self.make_envconfig(name, section, reader._subs, config) all_develop = all(name in config.envconfigs and config.envconfigs[name].usedevelop for name in config.envlist) config.skipsdist = reader.getbool("skipsdist", all_develop) def _list_section_factors(self, section): factors = set() if section in self._cfg: for _, value in self._cfg[section].items(): exprs = re.findall(r'^([\w{}\.,-]+)\:\s+', value, re.M) factors.update(*mapcat(_split_factor_expr, exprs)) return factors def make_envconfig(self, name, section, subs, config): factors = set(name.split('-')) reader = SectionReader(section, self._cfg, fallbacksections=["testenv"], factors=factors) vc = TestenvConfig(config=config, envname=name, factors=factors, reader=reader) reader.addsubstitutions(**subs) reader.addsubstitutions(envname=name) reader.addsubstitutions(envbindir=vc.get_envbindir, envsitepackagesdir=vc.get_envsitepackagesdir, envpython=vc.get_envpython) for env_attr in config._testenv_attr: atype = env_attr.type if atype in ("bool", "path", "string", "dict", "dict_setenv", "argv", "argvlist"): meth = getattr(reader, "get" + atype) res = meth(, env_attr.default) elif atype == "space-separated-list": res = reader.getlist(, sep=" ") elif atype == "line-list": res = reader.getlist(, sep="\n") else: raise ValueError("unknown type %r" % (atype,)) if env_attr.postprocess: res = env_attr.postprocess(testenv_config=vc, value=res) setattr(vc,, res) if atype == "path": reader.addsubstitutions(**{ res}) return vc def _getenvdata(self, reader): envstr = self.config.option.env \ or os.environ.get("TOXENV") \ or reader.getstring("envlist", replace=False) \ or [] envlist = _split_env(envstr) # collect section envs all_envs = set(envlist) - set(["ALL"]) for section in self._cfg: if all_envs.add([len(testenvprefix):]) if not all_envs: all_envs.add("python") if not envlist or "ALL" in envlist: envlist = sorted(all_envs) return envlist, all_envs def _split_env(env): """if handed a list, action="append" was used for -e """ if not isinstance(env, list): if '\n' in env: env = ','.join(env.split('\n')) env = [env] return mapcat(_expand_envstr, env) def _split_factor_expr(expr): partial_envs = _expand_envstr(expr) return [set(e.split('-')) for e in partial_envs] def _expand_envstr(envstr): # split by commas not in groups tokens = re.split(r'((?:\{[^}]+\})+)|,', envstr) envlist = [''.join(g).strip() for k, g in itertools.groupby(tokens, key=bool) if k] def expand(env): tokens = re.split(r'\{([^}]+)\}', env) parts = [token.split(',') for token in tokens] return [''.join(variant) for variant in itertools.product(*parts)] return mapcat(expand, envlist) def mapcat(f, seq): return list(itertools.chain.from_iterable(map(f, seq))) class DepConfig: def __init__(self, name, indexserver=None): = name self.indexserver = indexserver def __str__(self): if self.indexserver: if == "default": return return ":%s:%s" % (, return str( __repr__ = __str__ class IndexServerConfig: def __init__(self, name, url=None): = name self.url = url #: Check value matches substitution form #: of referencing value from other section. E.g. {[base]commands} is_section_substitution = re.compile("{\[[^{}\s]+\]\S+?}").match class SectionReader: def __init__(self, section_name, cfgparser, fallbacksections=None, factors=(), prefix=None): if prefix is None: self.section_name = section_name else: self.section_name = "%s:%s" % (prefix, section_name) self._cfg = cfgparser self.fallbacksections = fallbacksections or [] self.factors = factors self._subs = {} self._subststack = [] self._setenv = None def get_environ_value(self, name): if self._setenv is None: return os.environ.get(name) return self._setenv.get(name) def addsubstitutions(self, _posargs=None, **kw): self._subs.update(kw) if _posargs: self.posargs = _posargs def getpath(self, name, defaultpath): toxinidir = self._subs['toxinidir'] path = self.getstring(name, defaultpath) if path is not None: return toxinidir.join(path, abs=True) def getlist(self, name, sep="\n"): s = self.getstring(name, None) if s is None: return [] return [x.strip() for x in s.split(sep) if x.strip()] def getdict(self, name, default=None, sep="\n"): value = self.getstring(name, None) return self._getdict(value, default=default, sep=sep) def getdict_setenv(self, name, default=None, sep="\n"): value = self.getstring(name, None, replace=True, crossonly=True) definitions = self._getdict(value, default=default, sep=sep) self._setenv = SetenvDict(definitions, reader=self) return self._setenv def _getdict(self, value, default, sep): if value is None: return default or {} d = {} for line in value.split(sep): if line.strip(): name, rest = line.split('=', 1) d[name.strip()] = rest.strip() return d def getbool(self, name, default=None): s = self.getstring(name, default) if not s: s = default if s is None: raise KeyError("no config value [%s] %s found" % ( self.section_name, name)) if not isinstance(s, bool): if s.lower() == "true": s = True elif s.lower() == "false": s = False else: raise tox.exception.ConfigError( "boolean value %r needs to be 'True' or 'False'") return s def getargvlist(self, name, default=""): s = self.getstring(name, default, replace=False) return _ArgvlistReader.getargvlist(self, s) def getargv(self, name, default=""): return self.getargvlist(name, default)[0] def getstring(self, name, default=None, replace=True, crossonly=False): x = None for s in [self.section_name] + self.fallbacksections: try: x = self._cfg[s][name] break except KeyError: continue if x is None: x = default else: x = self._apply_factors(x) if replace and x and hasattr(x, 'replace'): x = self._replace(x, name=name, crossonly=crossonly) # print "getstring", self.section_name, name, "returned", repr(x) return x def _apply_factors(self, s): def factor_line(line): m ='^([\w{}\.,-]+)\:\s+(.+)', line) if not m: return line expr, line = m.groups() if any(fs <= self.factors for fs in _split_factor_expr(expr)): return line lines = s.strip().splitlines() return '\n'.join(filter(None, map(factor_line, lines))) def _replace(self, value, name=None, section_name=None, crossonly=False): if '{' not in value: return value section_name = section_name if section_name else self.section_name self._subststack.append((section_name, name)) try: return Replacer(self, crossonly=crossonly).do_replace(value) finally: assert self._subststack.pop() == (section_name, name) class Replacer: RE_ITEM_REF = re.compile( r''' (?<!\\)[{] (?:(?P<sub_type>[^[:{}]+):)? # optional sub_type for special rules (?P<substitution_value>(?:\[[^,{}]*\])?[^:,{}]*) # substitution key (?::(?P<default_value>[^{}]*))? # default value [}] ''', re.VERBOSE) def __init__(self, reader, crossonly=False): self.reader = reader self.crossonly = crossonly def do_replace(self, x): return self.RE_ITEM_REF.sub(self._replace_match, x) def _replace_match(self, match): g = match.groupdict() sub_value = g['substitution_value'] if self.crossonly: if sub_value.startswith("["): return self._substitute_from_other_section(sub_value) # in crossonly we return all other hits verbatim start, end = match.span() return match.string[start:end] # special case: opts and packages. Leave {opts} and # {packages} intact, they are replaced manually in # _venv.VirtualEnv.run_install_command. if sub_value in ('opts', 'packages'): return '{%s}' % sub_value try: sub_type = g['sub_type'] except KeyError: raise tox.exception.ConfigError( "Malformed substitution; no substitution type provided") if sub_type == "env": return self._replace_env(match) if sub_type is not None: raise tox.exception.ConfigError( "No support for the %s substitution type" % sub_type) return self._replace_substitution(match) def _replace_env(self, match): envkey ='substitution_value') if not envkey: raise tox.exception.ConfigError( 'env: requires an environment variable name') default ='default_value') envvalue = self.reader.get_environ_value(envkey) if envvalue is None: if default is None: raise tox.exception.ConfigError( "substitution env:%r: unknown environment variable %r " " or recursive definition." % (envkey, envkey)) return default return envvalue def _substitute_from_other_section(self, key): if key.startswith("[") and "]" in key: i = key.find("]") section, item = key[1:i], key[i + 1:] cfg = self.reader._cfg if section in cfg and item in cfg[section]: if (section, item) in self.reader._subststack: raise ValueError('%s already in %s' % ( (section, item), self.reader._subststack)) x = str(cfg[section][item]) return self.reader._replace(x, name=item, section_name=section, crossonly=self.crossonly) raise tox.exception.ConfigError( "substitution key %r not found" % key) def _replace_substitution(self, match): sub_key ='substitution_value') val = self.reader._subs.get(sub_key, None) if val is None: val = self._substitute_from_other_section(sub_key) if py.builtin.callable(val): val = val() return str(val) class _ArgvlistReader: @classmethod def getargvlist(cls, reader, value): """Parse ``commands`` argvlist multiline string. :param str name: Key name in a section. :param str value: Content stored by key. :rtype: list[list[str]] :raise :class:`tox.exception.ConfigError`: line-continuation ends nowhere while resolving for specified section """ commands = [] current_command = "" for line in value.splitlines(): line = line.rstrip() if not line: continue if line.endswith("\\"): current_command += " " + line[:-1] continue current_command += line if is_section_substitution(current_command): replaced = reader._replace(current_command, crossonly=True) commands.extend(cls.getargvlist(reader, replaced)) else: commands.append(cls.processcommand(reader, current_command)) current_command = "" else: if current_command: raise tox.exception.ConfigError( "line-continuation ends nowhere while resolving for [%s] %s" % (reader.section_name, "commands")) return commands @classmethod def processcommand(cls, reader, command): posargs = getattr(reader, "posargs", None) # Iterate through each word of the command substituting as # appropriate to construct the new command string. This # string is then broken up into exec argv components using # shlex. newcommand = "" for word in CommandParser(command).words(): if word == "{posargs}" or word == "[]": if posargs: newcommand += " ".join(posargs) continue elif word.startswith("{posargs:") and word.endswith("}"): if posargs: newcommand += " ".join(posargs) continue else: word = word[9:-1] new_arg = "" new_word = reader._replace(word) new_word = reader._replace(new_word) new_word = new_word.replace('\\{', '{').replace('\\}', '}') new_arg += new_word newcommand += new_arg # Construct shlex object that will not escape any values, # use all values as is in argv. shlexer = shlex.shlex(newcommand, posix=True) shlexer.whitespace_split = True shlexer.escape = '' argv = list(shlexer) return argv class CommandParser(object): class State(object): def __init__(self): self.word = '' self.depth = 0 self.yield_words = [] def __init__(self, command): self.command = command def words(self): ps = CommandParser.State() def word_has_ended(): return ((cur_char in string.whitespace and ps.word and ps.word[-1] not in string.whitespace) or (cur_char == '{' and ps.depth == 0 and not ps.word.endswith('\\')) or (ps.depth == 0 and ps.word and ps.word[-1] == '}') or (cur_char not in string.whitespace and ps.word and ps.word.strip() == '')) def yield_this_word(): yieldword = ps.word ps.word = '' if yieldword: ps.yield_words.append(yieldword) def yield_if_word_ended(): if word_has_ended(): yield_this_word() def accumulate(): ps.word += cur_char def push_substitution(): ps.depth += 1 def pop_substitution(): ps.depth -= 1 for cur_char in self.command: if cur_char in string.whitespace: if ps.depth == 0: yield_if_word_ended() accumulate() elif cur_char == '{': yield_if_word_ended() accumulate() push_substitution() elif cur_char == '}': accumulate() pop_substitution() else: yield_if_word_ended() accumulate() if ps.word.strip(): yield_this_word() return ps.yield_words def getcontextname(): if any(env in os.environ for env in ['JENKINS_URL', 'HUDSON_URL']): return 'jenkins' return None