example.py

import parsl
from parsl.providers import SlurmProvider

from parsl.config import Config
from parsl.executors.ipp import IPyParallelExecutor
from parsl.app.app import python_app

config = Config(
    executors=[
        IPyParallelExecutor(
            provider=SlurmProvider(
                'broadwl',
                walltime="01:00:00",
                init_blocks=1,
                max_blocks=64,
                nodes_per_block=1,
                tasks_per_node=32,
            ),
        )
    ]
)

parsl.set_stream_logger()
parsl.load(config)

@python_app
def inside_task(i, j):
    import psutil
    import os

    return '{}/{} is running on {}: {}%'.format(i, j, os.environ['HOSTNAME'], psutil.cpu_percent())

@python_app
def outside_task(inputs=[]):
    return '\n'.join(inputs)

outside_tasks = []
for i in range(0, 100):
    inside_tasks = []
    for j in range(0, 64):
        inside_tasks.append(inside_task(i, j))
    # outside_task will not be run until all of inside_tasks have completed
    outside_tasks.append(outside_task(inputs=inside_tasks))

with open('results.txt', 'w') as f:
    for t in outside_tasks:
        print(t.result(), file=f)