iterate_builddependencies.patch

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 34d0ba8e2..2dfe891bc 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -863,6 +863,8 @@ class EasyBlock(object):
         # otherwise we wipe the already partially populated installation directory,
         # see https://github.com/easybuilders/easybuild-framework/issues/2556
         if not (self.build_in_installdir and self.iter_idx > 0):
+            # make sure we no longer sit in the build directory before cleaning it.
+            change_dir(self.orig_workdir)
             self.make_dir(self.builddir, self.cfg['cleanupoldbuild'])
 
         trace_msg("build dir: %s" % self.builddir)
@@ -1478,7 +1480,7 @@ class EasyBlock(object):
         # this will only be done during first iteration, since after that the options won't be lists anymore
         for opt in ITERATE_OPTIONS:
             # keep track of list, supply first element as first option to handle
-            if isinstance(self.cfg[opt], (list, tuple)):
+            if isinstance(self.cfg[opt], (list, tuple)) and self.cfg[opt] and not isinstance(self.cfg[opt][0], dict):
                 self.iter_opts[opt] = self.cfg[opt]  # copy
                 self.log.debug("Found list for %s: %s", opt, self.iter_opts[opt])
 
@@ -1822,6 +1824,7 @@ class EasyBlock(object):
 
         # list of paths to include in RPATH filter;
         # only include builddir if we're not building in installation directory
+        self.rpath_filter_dirs = []
         self.rpath_filter_dirs.append(tempfile.gettempdir())
         if not self.build_in_installdir:
             self.rpath_filter_dirs.append(self.builddir)
@@ -1829,15 +1832,24 @@ class EasyBlock(object):
         # always include '<installdir>/lib', '<installdir>/lib64', $ORIGIN, $ORIGIN/../lib and $ORIGIN/../lib64
         # $ORIGIN will be resolved by the loader to be the full path to the executable or shared object
         # see also https://linux.die.net/man/8/ld-linux;
+        self.rpath_include_dirs = []
         self.rpath_include_dirs.append(os.path.join(self.installdir, 'lib'))
         self.rpath_include_dirs.append(os.path.join(self.installdir, 'lib64'))
         self.rpath_include_dirs.append('$ORIGIN')
         self.rpath_include_dirs.append('$ORIGIN/../lib')
         self.rpath_include_dirs.append('$ORIGIN/../lib64')
 
+        # in case of iterating builddependencies, unload any already loaded modules in reverse order
+        if 'builddependencies' in self.iter_opts:
+            if self.iter_idx > 0:
+                self.modules_tool.unload(reversed(self.loaded_modules))
+            # do not unload modules that were loaded before the toolchain preparation.
+            orig_modules = self.modules_tool.loaded_modules()
         # prepare toolchain: load toolchain module and dependencies, set up build environment
         self.toolchain.prepare(self.cfg['onlytcmod'], deps=self.cfg.dependencies(), silent=self.silent,
                                rpath_filter_dirs=self.rpath_filter_dirs, rpath_include_dirs=self.rpath_include_dirs)
+        if 'builddependencies' in self.iter_opts:
+            self.loaded_modules = self.modules_tool.loaded_modules()[len(orig_modules):]
 
         # keep track of environment variables that were tweaked and need to be restored after environment got reset
         # $TMPDIR may be tweaked for OpenMPI 2.x, which doesn't like long $TMPDIR paths...
@@ -2678,14 +2690,6 @@ class EasyBlock(object):
             """Return source step specified."""
             return get_step(SOURCE_STEP, "unpacking", source_substeps, True, initial=initial)
 
-        def prepare_step_spec(initial):
-            """Return prepare step specification."""
-            if initial:
-                substeps = [lambda x: x.prepare_step]
-            else:
-                substeps = [lambda x: x.guess_start_dir]
-            return (PREPARE_STEP, 'preparing', substeps, False)
-
         install_substeps = [
             (False, lambda x: x.stage_install_step),
             (False, lambda x: x.make_installdir),
@@ -2696,10 +2700,11 @@ class EasyBlock(object):
             """Return install step specification."""
             return get_step(INSTALL_STEP, "installing", install_substeps, True, initial=initial)
 
-        # format for step specifications: (stop_name: (description, list of functions, skippable))
+        # format for step specifications: (step_name, description, list of functions, skippable)
 
         # core steps that are part of the iterated loop
         patch_step_spec = (PATCH_STEP, 'patching', [lambda x: x.patch_step], True)
+        prepare_step_spec = (PREPARE_STEP, 'preparing', [lambda x: x.prepare_step], False)
         configure_step_spec = (CONFIGURE_STEP, 'configuring', [lambda x: x.configure_step], True)
         build_step_spec = (BUILD_STEP, 'building', [lambda x: x.build_step], True)
         test_step_spec = (TEST_STEP, 'testing', [lambda x: x.test_step], True)
@@ -2710,7 +2715,7 @@ class EasyBlock(object):
             ready_step_spec(True),
             source_step_spec(True),
             patch_step_spec,
-            prepare_step_spec(True),
+            prepare_step_spec,
             configure_step_spec,
             build_step_spec,
             test_step_spec,
@@ -2723,7 +2728,7 @@ class EasyBlock(object):
             ready_step_spec(False),
             source_step_spec(False),
             patch_step_spec,
-            prepare_step_spec(False),
+            prepare_step_spec,
             configure_step_spec,
             build_step_spec,
             test_step_spec,
diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py
index dd9bd1732..ff0181fbf 100644
--- a/easybuild/framework/easyconfig/easyconfig.py
+++ b/easybuild/framework/easyconfig/easyconfig.py
@@ -73,7 +73,7 @@ from easybuild.tools.systemtools import check_os_dependency
 from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME, DUMMY_TOOLCHAIN_VERSION
 from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA
 from easybuild.tools.toolchain.utilities import get_toolchain, search_toolchain
-from easybuild.tools.utilities import quote_py_str, remove_unwanted_chars
+from easybuild.tools.utilities import flatten, quote_py_str, remove_unwanted_chars
 from easybuild.tools.version import VERSION
 from easybuild.toolchains.compiler.cuda import Cuda
 
@@ -83,7 +83,8 @@ _log = fancylogger.getLogger('easyconfig.easyconfig', fname=False)
 MANDATORY_PARAMS = ['name', 'version', 'homepage', 'description', 'toolchain']
 
 # set of configure/build/install options that can be provided as lists for an iterated build
-ITERATE_OPTIONS = ['preconfigopts', 'configopts', 'prebuildopts', 'buildopts', 'preinstallopts', 'installopts']
+ITERATE_OPTIONS = ['builddependencies',
+                   'preconfigopts', 'configopts', 'prebuildopts', 'buildopts', 'preinstallopts', 'installopts']
 
 # name of easyconfigs archive subdirectory
 EASYCONFIGS_ARCHIVE_DIR = '__archive__'
@@ -174,7 +175,7 @@ def det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains,
 
 @toolchain_hierarchy_cache
 def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False):
-    """
+    r"""
     Determine list of subtoolchains for specified parent toolchain.
     Result starts with the most minimal subtoolchains first, ends with specified toolchain.
 
@@ -542,7 +543,11 @@ class EasyConfig(object):
         # parse dependency specifications
         # it's important that templating is still disabled at this stage!
         self.log.info("Parsing dependency specifications...")
-        self['builddependencies'] = [self._parse_dependency(dep, build_only=True) for dep in self['builddependencies']]
+        if self['builddependencies'] and isinstance(self['builddependencies'][0][0], (list,tuple)):
+            self['builddependencies'] = [[self._parse_dependency(dep, build_only=True) for dep in x]
+                                         for x in self['builddependencies']]
+        else:
+            self['builddependencies'] = [self._parse_dependency(dep, build_only=True) for dep in self['builddependencies']]
         self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']]
         self['hiddendependencies'] = [self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']]
 
@@ -665,7 +670,7 @@ class EasyConfig(object):
                 raise EasyBuildError("%s not available in self.cfg (anymore)?!", opt)
 
             # keep track of list, supply first element as first option to handle
-            if isinstance(self[opt], (list, tuple)):
+            if isinstance(self[opt], (list, tuple)) and self[opt] and not isinstance(self[opt][0], dict):
                 opt_counts.append((opt, len(self[opt])))
 
         # make sure that options that specify lists have the same length
@@ -679,13 +684,14 @@ class EasyConfig(object):
         """
         Filter hidden dependencies from list of (build) dependencies.
         """
-        dep_mod_names = [dep['full_mod_name'] for dep in self['dependencies'] + self['builddependencies']]
-        build_dep_mod_names = [dep['full_mod_name'] for dep in self['builddependencies']]
+        dep_mod_names = [dep['full_mod_name'] for dep in self['dependencies'] + self.builddependencies()]
+        build_dep_mod_names = [dep['full_mod_name'] for dep in self.builddependencies()]
 
         faulty_deps = []
         for i, hidden_dep in enumerate(self['hiddendependencies']):
             hidden_mod_name = ActiveMNS().det_full_module_name(hidden_dep)
             visible_mod_name = ActiveMNS().det_full_module_name(hidden_dep, force_visible=True)
+            iterset = set()
 
             # track whether this hidden dep is listed as a build dep
             if visible_mod_name in build_dep_mod_names or hidden_mod_name in build_dep_mod_names:
@@ -694,16 +700,33 @@ class EasyConfig(object):
                 enable_templating = self.enable_templating
                 self.enable_templating = False
                 self['hiddendependencies'][i]['build_only'] = True
+                self['hiddendependencies'][i]['iterations'] = iterset
                 self.enable_templating = enable_templating
 
             # filter hidden dep from list of (build)dependencies
             if visible_mod_name in dep_mod_names:
                 for key in ['builddependencies', 'dependencies']:
-                    self[key] = [d for d in self[key] if d['full_mod_name'] != visible_mod_name]
+                    if self[key] and not isinstance(self[key][0], dict):
+                        val = [[d for d in deps if d['full_mod_name'] != visible_mod_name]
+                               for deps in self[key]]
+                        for idx, deps in enumerate(val):
+                            if len(deps) < len(self[key][idx]):
+                                iterset.add(idx)
+                        self[key] = val
+                    else:
+                        self[key] = [d for d in self[key] if d['full_mod_name'] != visible_mod_name]
                 self.log.debug("Removed (build)dependency matching hidden dependency %s", hidden_dep)
             elif hidden_mod_name in dep_mod_names:
                 for key in ['builddependencies', 'dependencies']:
-                    self[key] = [d for d in self[key] if d['full_mod_name'] != hidden_mod_name]
+                    if self[key] and not isinstance(self[key][0], dict):
+                        val = [[d for d in deps if d['full_mod_name'] != hidden_mod_name]
+                               for deps in self[key]]
+                        for idx, deps in enumerate(val):
+                            if len(deps) < len(self[key][idx]):
+                                iterset.add(idx)
+                        self[key] = val
+                    else:
+                        self[key] = [d for d in self[key] if d['full_mod_name'] != hidden_mod_name]
                 self.log.debug("Hidden (build)dependency %s is already marked to be installed as a hidden module",
                                hidden_dep)
             else:
@@ -809,10 +832,9 @@ class EasyConfig(object):
 
         :param build_only: only return build dependencies, discard others
         """
-        if build_only:
-            deps = self['builddependencies']
-        else:
-            deps = self['dependencies'] + self['builddependencies'] + self['hiddendependencies']
+        deps = self.builddependencies()
+        if not build_only:
+            deps = self['dependencies'] + deps + self['hiddendependencies']
 
         # if filter-deps option is provided we "clean" the list of dependencies for
         # each processed easyconfig to remove the unwanted dependencies
@@ -827,7 +849,11 @@ class EasyConfig(object):
         """
         return the parsed build dependencies
         """
-        return self['builddependencies']
+        builddeps = self['builddependencies']
+        if builddeps and not isinstance(builddeps[0], dict):
+            # flatten if not iterating yet
+            builddeps = flatten(builddeps)
+        return builddeps
 
     @property
     def name(self):
@@ -1096,10 +1122,15 @@ class EasyConfig(object):
         for key in DEPENDENCY_PARAMETERS:
             # loop over a *copy* of dependency dicts (with resolved templates);
             # to update the original dep dict, we need to index with idx into self._config[key][0]...
-            for idx, dep in enumerate(self[key]):
+            val = self[key]
+            orig_val = self._config[key][0]
+            if val and not isinstance(val[0], dict):
+                val = flatten(val)
+                orig_val = flatten(orig_val)
+            for idx, dep in enumerate(val):
 
                 # reference to original dep dict, this is the one we should be updating
-                orig_dep = self._config[key][0][idx]
+                orig_dep = orig_val[idx]
 
                 if filter_deps and orig_dep['name'] in filter_deps:
                     self.log.debug("Skipping filtered dependency %s when finalising dependencies", orig_dep['name'])
diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py
index 21483101e..aa9342bb7 100644
--- a/easybuild/framework/easyconfig/format/one.py
+++ b/easybuild/framework/easyconfig/format/one.py
@@ -52,6 +52,7 @@ EB_FORMAT_EXTENSION = '.eb'
 # dependency parameters always need to be reformatted, to correctly deal with dumping parsed dependencies
 REFORMAT_FORCED_PARAMS = ['sanity_check_paths'] + DEPENDENCY_PARAMETERS
 REFORMAT_SKIPPED_PARAMS = ['toolchain', 'toolchainopts']
+REFORMAT_LIST_OF_LISTS_OF_TUPLES = ['builddependencies']
 REFORMAT_THRESHOLD_LENGTH = 100  # only reformat lines that would be longer than this amount of characters
 REFORMAT_ORDERED_ITEM_KEYS = {
     'sanity_check_paths': ['files', 'dirs'],
@@ -140,6 +141,7 @@ class FormatOneZero(EasyConfigFormatConfigObj):
         # note: this does not take into account the parameter name + '=', only the value
         line_too_long = len(param_strval) + addlen > REFORMAT_THRESHOLD_LENGTH
         forced = param_name in REFORMAT_FORCED_PARAMS
+        list_of_lists_of_tuples = param_name in REFORMAT_LIST_OF_LISTS_OF_TUPLES
 
         if param_name in REFORMAT_SKIPPED_PARAMS:
             self.log.info("Skipping reformatting value for parameter '%s'", param_name)
@@ -170,9 +172,15 @@ class FormatOneZero(EasyConfigFormatConfigObj):
                     for item in param_val:
                         comment = self._get_item_comments(param_name, item).get(str(item), '')
                         addlen = addlen + len(INDENT_4SPACES) + len(comment)
+                        if isinstance(item, list) and list_of_lists_of_tuples:
+                            itemstr = '[' + (",\n " + INDENT_4SPACES).join([
+                                self._reformat_line(param_name, subitem, outer=True, addlen=addlen)
+                                for subitem in item]) + ']'
+                        else:
+                            itemstr = self._reformat_line(param_name, item, addlen=addlen)
                         res += item_tmpl % {
                             'comment': comment,
-                            'item': self._reformat_line(param_name, item, addlen=addlen)
+                            'item': itemstr
                         }
 
                 # end with closing character: ], ), }
@@ -230,11 +238,18 @@ class FormatOneZero(EasyConfigFormatConfigObj):
                 if key == 'dependencies':
                     val.extend([d for d in ecfg['hiddendependencies'] if not d['build_only']])
                 elif key == 'builddependencies':
-                    val.extend([d for d in ecfg['hiddendependencies'] if d['build_only']])
+                    if val and not isinstance(val[0], dict):
+                        for idx, deps in enumerate(val):
+                            val[idx].extend([d for d in ecfg['hiddendependencies'] if d['build_only'] and
+                                             idx in d['iterations']])
+                    else:
+                        val.extend([d for d in ecfg['hiddendependencies'] if d['build_only']])
 
                 if val != default_values[key]:
                     # dependency easyconfig parameters were parsed, so these need special care to 'unparse' them
-                    if key in DEPENDENCY_PARAMETERS:
+                    if key == 'builddependencies' and val and not isinstance(val[0], dict):
+                        valstr = [[dump_dependency(d, ecfg['toolchain']) for d in dep] for dep in val]
+                    elif key in DEPENDENCY_PARAMETERS:
                         valstr = [dump_dependency(d, ecfg['toolchain']) for d in val]
                     elif key == 'toolchain':
                         valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[key]
diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py
index 627811d12..a772926b6 100644
--- a/easybuild/framework/easyconfig/tweak.py
+++ b/easybuild/framework/easyconfig/tweak.py
@@ -56,7 +56,7 @@ from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
 from easybuild.tools.robot import resolve_dependencies, robot_find_easyconfig
 from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME
 from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES
-from easybuild.tools.utilities import quote_str
+from easybuild.tools.utilities import flatten, quote_str
 
 
 _log = fancylogger.getLogger('easyconfig.tweak', fname=False)
@@ -832,9 +832,14 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=
     for key in DEPENDENCY_PARAMETERS:
         # loop over a *copy* of dependency dicts (with resolved templates);
         # to update the original dep dict, we need to index with idx into self._config[key][0]...
-        for idx, dep in enumerate(parsed_ec['ec'][key]):
+        val = parsed_ec['ec'][key]
+        orig_val = parsed_ec['ec']._config[key][0]
+        if val and not isinstance(val[0], dict):
+            val = flatten(val)
+            orig_val = flatten(orig_val)
+        for idx, dep in enumerate(val):
             # reference to original dep dict, this is the one we should be updating
-            orig_dep = parsed_ec['ec']._config[key][0][idx]
+            orig_dep = orig_val[idx]
             # skip dependencies that are marked as external modules
             if dep['external_module']:
                 continue
diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb
index fe6e2840f..b680ad8b2 100644
--- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb
+++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb
@@ -5,19 +5,26 @@ versionsuffix = '-iter'
 homepage = 'https://easybuilders.github.io/easybuild'
 description = "Toy C program, 100% toy."
 
-toolchain = {'name': 'dummy', 'version': 'dummy'}
+toolchain = {'name': 'dummy', 'version': ''}
 
 sources = [SOURCE_TAR_GZ]
 patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch']
 
+builddependencies = [
+    [('GCC', '6.4.0-2.28'),
+     ('OpenMPI', '2.1.2', '', ('GCC', '6.4.0-2.28'))],
+    [('GCC', '6.4.0-2.28')],
+    [('GCC', '7.3.0-2.30')],
+]
+
 buildopts = [
     '',
-    "-O2; mv %(name)s toy_O2",
-    "-O1; mv %(name)s toy_O1",
+    "-O2; mv %(name)s toy_O2_$EBVERSIONGCC",
+    "-O1; mv %(name)s toy_O1_$EBVERSIONGCC",
 ]
 
 sanity_check_paths = {
-    'files': [('bin/yot', 'bin/toy'), 'bin/toy_O1', 'bin/toy_O2'],
+    'files': [('bin/yot', 'bin/toy'), 'bin/toy_O1_7.3.0-2.30', 'bin/toy_O2_6.4.0-2.28'],
     'dirs': ['bin'],
 }
 
diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
index ff1ec1c7a..5bf8bb23d 100644
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -1796,7 +1796,7 @@ class ToyBuildTest(EnhancedTestCase):
         topdir = os.path.abspath(os.path.dirname(__file__))
         toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-iter.eb')
 
-        expected_buildopts = ['', '-O2; mv %(name)s toy_O2', '-O1; mv %(name)s toy_O1']
+        expected_buildopts = ['', '-O2; mv %(name)s toy_O2_$EBVERSIONGCC', '-O1; mv %(name)s toy_O1_$EBVERSIONGCC']
 
         for extra_args in [None, ['--minimal-toolchains']]:
             # sanity check will make sure all entries in buildopts list were taken into account