[med-svn] [python-ruffus] 03/05: Imported Upstream version 2.6.2+dfsg
Andreas Tille
tille at debian.org
Thu Mar 12 20:52:31 UTC 2015
This is an automated email from the git hooks/post-receive script.
tille pushed a commit to branch master
in repository python-ruffus.
commit ac493a8a5273afd3df323db9c5bd154677deabcb
Author: Andreas Tille <tille at debian.org>
Date: Thu Mar 12 21:50:26 2015 +0100
Imported Upstream version 2.6.2+dfsg
---
CHANGES.TXT | 78 +
PKG-INFO | 2 +-
doc/Makefile | 2 +-
doc/_build/latex/ruffus.pdf | Bin 4513083 -> 4688129 bytes
doc/_templates/index.html | 23 +-
doc/_templates/layout.html | 2 +-
doc/contents.rst | 3 +
doc/decorators/active_if.rst | 2 +-
doc/decorators/check_if_uptodate.rst | 1 +
doc/decorators/collate.rst | 105 +-
doc/decorators/collate_ex.rst | 105 +-
doc/decorators/combinations.rst | 239 +-
doc/decorators/combinations_with_replacement.rst | 238 +-
doc/decorators/files.rst | 6 +-
doc/decorators/files_ex.rst | 2 +-
doc/decorators/follows.rst | 2 +-
doc/decorators/graphviz.rst | 1 +
doc/decorators/indicator_objects.rst | 5 +-
doc/decorators/jobs_limit.rst | 2 +-
doc/decorators/merge.rst | 46 +-
doc/decorators/mkdir.rst | 59 +-
doc/decorators/originate.rst | 40 +-
doc/decorators/parallel.rst | 2 +-
doc/decorators/permutations.rst | 232 +-
doc/decorators/posttask.rst | 1 +
doc/decorators/product.rst | 295 +-
doc/decorators/split.rst | 68 +-
doc/decorators/subdivide.rst | 45 +-
doc/decorators/transform.rst | 72 +-
doc/decorators/transform_ex.rst | 136 +-
doc/examples/bioinformatics/part2.rst | 2 +-
doc/faq.rst | 36 +-
doc/global.inc | 11 +-
doc/history.rst | 174 +-
doc/images/small_logo.png | Bin 0 -> 14521 bytes
doc/images/subpipeline_example.png | Bin 0 -> 65279 bytes
doc/implementation_notes.rst | 434 +-
doc/installation.rst | 61 +-
.../example_scripts/play_with_colours.py | 2 +-
doc/static_data/example_scripts/simpler.py | 46 +-
doc/static_data/ruffus.pdf | Bin 4513083 -> 4688129 bytes
doc/static_data/small_logo.png | Bin 0 -> 14521 bytes
doc/todo.rst | 226 +-
doc/tutorials/new_syntax.rst | 485 ++
doc/tutorials/new_syntax_worked_example.rst | 355 ++
doc/tutorials/new_syntax_worked_example_code.rst | 222 +
doc/tutorials/new_tutorial/checkpointing_code.rst | 36 +-
.../new_tutorial/flowchart_colours_code.rst | 2 +-
doc/tutorials/new_tutorial/parallel.rst | 1 +
.../new_tutorial/pipeline_printout_graph.rst | 3 +
ruffus/__init__.py | 23 +-
ruffus/cmdline.py | 6 +-
ruffus/drmaa_wrapper.py | 50 +-
ruffus/file_name_parameters.py | 218 +-
ruffus/graph.py | 299 +-
ruffus/parse_old_style_ruffus.py | 268 +
ruffus/print_dependencies.py | 37 +-
ruffus/proxy_logger.py | 2 +
ruffus/ruffus_exceptions.py | 42 +-
ruffus/ruffus_utility.py | 456 +-
ruffus/ruffus_version.py | 2 +-
ruffus/task.py | 5968 +++++++++++++-------
.../auto_generated_pipeline_examples/parallel.py | 4 +-
.../auto_generated_pipeline_examples/simple.py | 4 +-
.../auto_generated_pipeline_examples/simpler.py | 4 +-
.../create_test_script_from_dependency_tree.py | 78 +-
ruffus/test/draw_specified_dependency_tree.py | 40 +-
...eptions.py => manual_test_ctrl_c_exceptions.py} | 0
.../test/{test_drmaa.py => manual_test_drmaa.py} | 0
ruffus/test/play_with_colours.py | 2 +-
ruffus/test/simpler_at_runtime.py | 253 -
ruffus/test/simpler_with_shared_logging.py | 2 +-
ruffus/test/test_N_x_M_and_collate.py | 223 +-
ruffus/test/test_active_if.py | 165 +-
ruffus/test/test_branching_dependencies.py | 392 +-
ruffus/test/test_cmdline.py | 47 +-
ruffus/test/test_collate.py | 223 +-
ruffus/test/test_combinatorics.py | 75 +-
ruffus/test/test_empty_files_decorator.py | 184 +-
ruffus/test/test_exceptions.py | 69 +-
ruffus/test/test_file_name_parameters.py | 387 +-
ruffus/test/test_files_decorator.py | 186 +-
ruffus/test/test_files_post_merge.py | 299 -
ruffus/test/test_filesre_combine.py | 191 +-
ruffus/test/test_filesre_split_and_combine.py | 257 +-
ruffus/test/test_follows_mkdir.py | 180 +-
ruffus/test/test_graphviz.py | 138 +-
..._inputs_with_multiple_args_raising_exception.py | 198 +-
ruffus/test/test_job_completion_checksums.py | 404 +-
ruffus/test/test_job_history_with_exceptions.py | 170 +-
ruffus/test/test_mkdir.py | 66 +-
...inatorics.py => test_newstyle_combinatorics.py} | 306 +-
ruffus/test/test_newstyle_proxy.py | 181 +
...es.py => test_newstyle_regex_error_messages.py} | 197 +-
ruffus/test/test_pausing.py | 294 +-
ruffus/test/test_pipeline_printout_graph.py | 205 +
ruffus/test/test_posttask_merge.py | 244 +
ruffus/test/test_regex_error_messages.py | 48 +-
ruffus/test/test_ruffus_utility.py | 170 +-
.../test_ruffus_utility_parse_task_arguments.py | 713 +++
ruffus/test/test_runtime_data.py | 187 +
ruffus/test/test_softlink_uptodate.py | 151 +-
ruffus/test/test_split_and_combine.py | 287 +-
ruffus/test/test_split_regex_and_collate.py | 115 +-
ruffus/test/test_split_subdivide_checkpointing.py | 239 +
ruffus/test/test_subpipeline.py | 201 +
ruffus/test/test_suffix_output_dir.py | 259 +
ruffus/test/test_task_file_dependencies.py | 130 +-
ruffus/test/test_task_misc.py | 63 +-
ruffus/test/test_transform_add_inputs.py | 234 +-
ruffus/test/test_transform_inputs.py | 207 +-
ruffus/test/test_transform_with_no_re_matches.py | 194 +-
ruffus/test/test_tutorial7.py | 137 +-
ruffus/test/test_unicode_filenames.py | 185 +-
ruffus/test/test_verbosity.py | 137 +-
115 files changed, 13423 insertions(+), 7183 deletions(-)
diff --git a/CHANGES.TXT b/CHANGES.TXT
index 85bd8d6..786d3f5 100644
--- a/CHANGES.TXT
+++ b/CHANGES.TXT
@@ -1,3 +1,81 @@
+= v. 2.6=
+ 2015-03-12
+ ============================================================================================================================================================
+ Bug fixes
+ ============================================================================================================================================================
+
+ * ``pipeline_printout_graph()`` incompatibility with python3 fixed
+ * checkpointing did not work correctly with :ref:`@split(...) <decorators.split>` and :ref:`@subdivide(...) <decorators.subdivide>`
+ =====================================================================================================================
+ `@transform `(..., suffix("xxx"),` :red:`output_dir` `= "/new/output/path")`
+ =====================================================================================================================
+
+ * Thanks to the suggestion of Milan Simonovic.
+ * :ref:`@transform <decorators.transform>` takes an optional ``output_dir`` named parameter with :ref:`suffix() <new_manual.suffix>`.
+ * Output files go to the specified directory.
+
+ =====================================================================================================================
+ Named parameters
+ =====================================================================================================================
+
+ * Decorators can take named parameters.
+ * These are self documenting, and improve clarity.
+
+ =============================================
+ New object orientated syntax for Ruffus
+ =============================================
+
+ * Ruffus Pipelines can now be created directly using the new ``Pipeline`` and ``Task`` objects instead of via decorators.
+ * This new syntax is fully compatible and inter-operates with traditional Ruffus syntax using decorators.
+ * Apart from cosmetic changes, the new syntax allows different instances of modular Ruffus sub-pipelines
+ to be defined separately, in different python modules and then joined together flexible at runtime.
+
+= v. 2.5=
+ 2014-08-06
+
+ ============================================================================================================================================================
+ Python3 compatability
+ ============================================================================================================================================================
+
+ At least python 2.6 is now required.
+
+ ============================================================================================================================================================
+ Ctrl-C interrupts
+ ============================================================================================================================================================
+
+ Ruffus now mostly(!) terminates gracefully when interrupted by Ctrl-C .
+
+ ============================================================================================================================================================
+ Customising flowcharts in pipeline_printout_graph() with ``@graphviz``
+ ============================================================================================================================================================
+
+ *Contributed by Sean Davis, with improved syntax via Jake Biesinger*
+
+ The graphics for each task can have its own attributes (URL, shape, colour) etc. by adding
+ `graphviz attributes <http://www.graphviz.org/doc/info/attrs.html>`__
+ using the ``@graphviz`` decorator.
+
+ ============================================================================================================================================================
+ Consistent verbosity levels
+ ============================================================================================================================================================
+
+ The verbosity levels are now more fine-grained and consistent between pipeline_printout and pipeline_run.
+
+ ============================================================================================================================================================
+ Allow abbreviated paths from ``pipeline_run`` or ``pipeline_printout``
+ ============================================================================================================================================================
+
+ Ruffus now allows either
+ 1) Only the nth top level sub-directories to be included
+ 2) The message to be truncated to a specified number of characters (to fit on a line, for example)
+
+ ============================================================================================================================================================
+ Bug fixes
+ ============================================================================================================================================================
+ * BUG FIX: Output producing wild cards was not saved in the checksum files!!!
+ * BUG FIX: @mkdir bug under Windows. Thanks to Sean Turley. (Aargh! Different exceptions are thrown in Windows vs. Linux for the same condition!)
+ * Added :ref:`pipeline_get_task_names(...) <pipeline_functions.pipeline_get_task_names>` which returns all task name as a list of strings. Thanks to Clare Sloggett
+
= v. 2.4.1=
2014-04-26
* Breaking changes to drmaa API suggested by Bernie Pope to ensure portability across different drmaa implementations (SGE, SLURM etc.)
diff --git a/PKG-INFO b/PKG-INFO
index 2733964..08e8ac3 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: ruffus
-Version: 2.5
+Version: 2.6.2
Summary: Light-weight Python Computational Pipeline Management
Home-page: http://www.ruffus.org.uk
Author: Leo Goodstadt
diff --git a/doc/Makefile b/doc/Makefile
index 322cc5b..53499b9 100644
--- a/doc/Makefile
+++ b/doc/Makefile
@@ -4,7 +4,7 @@
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
-PAPER =
+PAPER = a4
BUILDDIR = _build
# User-friendly check for sphinx-build
diff --git a/doc/_build/latex/ruffus.pdf b/doc/_build/latex/ruffus.pdf
index 68836b8..0c4877e 100644
Binary files a/doc/_build/latex/ruffus.pdf and b/doc/_build/latex/ruffus.pdf differ
diff --git a/doc/_templates/index.html b/doc/_templates/index.html
index 4ade6f5..487ceb5 100644
--- a/doc/_templates/index.html
+++ b/doc/_templates/index.html
@@ -1,6 +1,20 @@
{% extends "layout.html" %}
{% set title = 'Ruffus' %}
+
+{% block extrahead %}
+<meta content='Ruffus is a Computation Pipeline library for python. It is open-sourced, powerful and user-friendly, and widely used in science and bioinformatics' name='description'/>
+<meta content='python, pipeline, computation, cluster, Ruffus, bioinformatics, parallel' name='keywords'/>
+<meta content='Leo Goodstadt' name='Author'/>
+<meta content='Leo Goodstadt' name='Publisher'/>
+<meta content='ruffus at llew.org.uk' name='Email'/>
+<meta content='index, follow' name='robots'/>
+<meta content='en-uk' name='language'/>
+<meta content='United Kingdom' name='country'/>
+<meta name='copyright' content='Leo Goodstadt'/>
+{% endblock %}
+
+
{% block body %}
<p>
@@ -9,7 +23,14 @@
Ruffus is a Computation Pipeline library for python. It is open-sourced,
-powerful and user-friendly, and widely used in science and bioinformatics.
+powerful and user-friendly, and widely used in science and bioinformatics.<br/><br/>
+
+<h1>Citation:</h1>
+<blockquote>
+Please cite <I>Ruffus</I> as:<br/>
+<div><a class="reference external" href="http://bioinformatics.oxfordjournals.org/content/early/2010/09/16/bioinformatics.btq524">Leo Goodstadt (2010)</a> :
+<strong>Ruffus: a lightweight Python library for computational pipelines.</strong> <em>Bioinformatics</em> 26(21): 2778-2779</div></blockquote><br/>
+
<h1>Welcome</h1>
diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html
index 554a7c9..144c0e2 100644
--- a/doc/_templates/layout.html
+++ b/doc/_templates/layout.html
@@ -1,5 +1,4 @@
{% extends "!layout.html" %}
-
{% block sidebarsearch %}
{{ super() }}
<h3> Quick Reference:</h3>
@@ -68,6 +67,7 @@
<li><a href="{{ pathto('cheatsheet') }}">Cheat sheet</a> | </li>
<li><a href="{{ pathto('tutorials/new_tutorial/command_line') }}">Command Line</a> | </li>
<li><a href="{{ pathto('gallery') }}">Gallery</a> | </li>
+ <li><a href="{{ pathto('tutorials/new_syntax') }}">New syntax in v 2.6</a> | </li>
<li><a href="{{ pathto('history') }}">Latest Changes</a> » </li>
{% endblock %}
diff --git a/doc/contents.rst b/doc/contents.rst
index 0e91439..1d348a1 100644
--- a/doc/contents.rst
+++ b/doc/contents.rst
@@ -79,6 +79,9 @@ Overview:
installation.rst
design.rst
Bugs and Updates <history>
+ tutorials/new_syntax.rst
+ tutorials/new_syntax_worked_example.rst
+ tutorials/new_syntax_worked_example_code.rst
Future plans <todo>
Implementation_notes <implementation_notes.rst>
faq.rst
diff --git a/doc/decorators/active_if.rst b/doc/decorators/active_if.rst
index 1b02e1f..5f14ee6 100644
--- a/doc/decorators/active_if.rst
+++ b/doc/decorators/active_if.rst
@@ -5,8 +5,8 @@
.. seealso::
+ * :ref:`@active_if <new_manual.active_if>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
- * More on @active_if in the ``Ruffus`` :ref:`Manual <new_manual.active_if>`
############
diff --git a/doc/decorators/check_if_uptodate.rst b/doc/decorators/check_if_uptodate.rst
index 893f430..ed93096 100644
--- a/doc/decorators/check_if_uptodate.rst
+++ b/doc/decorators/check_if_uptodate.rst
@@ -6,6 +6,7 @@
.. seealso::
+ * :ref:`@check_if_uptodate <new_manual.check_if_uptodate>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
.. |dependency_checking_function| replace:: `dependency_checking_function`
diff --git a/doc/decorators/collate.rst b/doc/decorators/collate.rst
index 1121925..4328f97 100644
--- a/doc/decorators/collate.rst
+++ b/doc/decorators/collate.rst
@@ -3,74 +3,89 @@
.. index::
pair: @collate; Syntax
+.. role:: raw-html(raw)
+ :format: html
+
+:raw-html:`<style> .red {color:red} </style>`
+
+.. role:: red
+
.. seealso::
+ * :ref:`@collate <new_manual.collate>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
-########################
- at collate
-########################
-
-.. |tasks_or_file_names| replace:: `tasks_or_file_names`
-.. _tasks_or_file_names: `decorators.collate.tasks_or_file_names`_
-.. |extra_parameters| replace:: `extra_parameters`
-.. _extra_parameters: `decorators.collate.extra_parameters`_
-.. |output_pattern| replace:: `output_pattern`
-.. _output_pattern: `decorators.collate.output_pattern`_
+.. |input| replace:: `input`
+.. _input: `decorators.collate.input`_
+.. |extras| replace:: `extras`
+.. _extras: `decorators.collate.extras`_
+.. |output| replace:: `output`
+.. _output: `decorators.collate.output`_
+.. |filter| replace:: `filter`
+.. _filter: `decorators.collate.filter`_
.. |matching_regex| replace:: `matching_regex`
.. _matching_regex: `decorators.collate.matching_regex`_
.. |matching_formatter| replace:: `matching_formatter`
.. _matching_formatter: `decorators.collate.matching_formatter`_
-********************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
-*@collate* ( |tasks_or_file_names|_, :ref:`regex<decorators.regex>`\ *(*\ |matching_regex|_\ *)* | :ref:`formatter<decorators.formatter>`\ *(*\ |matching_formatter|_\ *)*\, |output_pattern|_, [|extra_parameters|_,...] )
-********************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
- **Purpose:**
- Groups / collates sets of input files, each into a separate summary.
+########################################################################
+ at collate( |input|_, |filter|_, |output|_, [|extras|_,...] )
+########################################################################
+
+
- Only out of date tasks (comparing input and output files) will be run
+ **Purpose:**
- Output file names and strings in the extra parameters
- are determined from |tasks_or_file_names|_, i.e. from the output
- of up stream tasks, or a list of file names.
+ Use |filter|_ to identify common sets of |input|_\s which are to be grouped or collated together:
- String replacement occurs either through suffix matches via :ref:`suffix<decorators.suffix>` or
- the :ref:`formatter<decorators.formatter>` or :ref:`regex<decorators.regex>` indicators.
+ Each set of |input|_\ s which generate identical |output|_ and |extras|_ using the
+ :ref:`formatter<decorators.formatter>` or :ref:`regex<decorators.regex>` (regular expression)
+ filters are collated into one job.
- ``@collate`` groups together all **Input** which result in identical **Output** and **extra**
- parameters.
+ This is a **many to fewer** operation.
- It is a **many to fewer** operation.
+ Only out of date jobs (comparing input and output files) will be re-run.
**Example**:
- ``regex(r".*(\..+)"), "\1.summary"`` creates a separate summary file for each suffix::
+ ``regex(r".+\.(.+)$")``, ``"\1.summary"`` creates a separate summary file for each suffix::
animal_files = "a.fish", "b.fish", "c.mammals", "d.mammals"
# summarise by file suffix:
- @collate(animal_files, regex(r"\.(.+)$"), r'\1.summary')
+ @collate(animal_files, regex(r".+\.(.+)$"), r'\1.summary')
def summarize(infiles, summary_file):
pass
+
+ #. |output|_ and optional |extras|_ parameters are passed to the functions after string
+ substitution. Non-string values are passed through unchanged.
+ #. Each collate job consists of |input|_ files which are aggregated by string substitution
+ to identical |output|_ and |extras|_
+ #. | The above example results in two jobs:
+ | ``["a.fish", "b.fish" -> "fish.summary"]``
+ | ``["c.mammals", "d.mammals" -> "mammals.summary"]``
+
**Parameters:**
-.. _decorators.collate.tasks_or_file_names:
+.. _decorators.collate.input:
- * *tasks_or_file_names*
+ * **input** = *tasks_or_file_names*
can be a:
- #. Task / list of tasks (as in the example above).
+ #. Task / list of tasks.
File names are taken from the output of the specified task(s)
- #. (Nested) list of file name strings.
+ #. (Nested) list of file name strings (as in the example above).
File names containing ``*[]?`` will be expanded as a |glob|_.
E.g.:``"a.*" => "a.1", "a.2"``
+.. _decorators.collate.filter:
+
.. _decorators.collate.matching_regex:
- * *matching_regex*
+ * **filter** = *matching_regex*
is a python regular expression string, which must be wrapped in
a :ref:`regex<decorators.regex>` indicator object
See python `regular expression (re) <http://docs.python.org/library/re.html>`_
@@ -78,30 +93,26 @@
.. _decorators.collate.matching_formatter:
- * *matching_formatter*
+ * **filter** = *matching_formatter*
a :ref:`formatter<decorators.formatter>` indicator object containing optionally
a python `regular expression (re) <http://docs.python.org/library/re.html>`_.
-.. _decorators.collate.output_pattern:
+.. _decorators.collate.output:
+
+ * **output** = *output*
+ Specifies the resulting output file name(s) after string substitution
+
+.. _decorators.collate.extras:
+
+ * **extras** = *extras*
+ Any extra parameters are passed verbatim to the task function
- * *output_pattern*
- Specifies the resulting output file name(s).
+ If you are using named parameters, these can be passed as a list, i.e. ``extras= [...]``
-.. _decorators.collate.extra_parameters:
+ Any extra parameters are consumed by the task function and not forwarded further down the pipeline.
- * *extra_parameters*
- Any extra parameters are passed verbatim to the task function
- #. *outputs* and optional extra parameters are passed to the functions after string
- substitution in any strings. Non-string values are passed through unchanged.
- #. Each collate job consists of input files which are aggregated by string substitution
- to a single set of output / extra parameter matches
- #. In the above cases, ``a.fish`` and ``b.fish`` both produce ``fish.summary`` after regular
- expression subsitution, and are collated into a single job:
- ``["a.fish", "b.fish" -> "fish.summary"]``
- while ``c.mammals``, ``d.mammals`` both produce ``mammals.summary``, are collated in a separate job:
- ``["c.mammals", "d.mammals" -> "mammals.summary"]``
**Example2**:
diff --git a/doc/decorators/collate_ex.rst b/doc/decorators/collate_ex.rst
index 50175f4..421af49 100644
--- a/doc/decorators/collate_ex.rst
+++ b/doc/decorators/collate_ex.rst
@@ -5,47 +5,58 @@
pair: @collate, inputs(...); Syntax
pair: @collate, add_inputs(...); Syntax
+.. role:: raw-html(raw)
+ :format: html
+
+:raw-html:`<style> .red {color:red} </style>`
+
+.. role:: red
+
+
.. seealso::
+ * :ref:`@collate <new_manual.collate>` in the **Ruffus** Manual
+ * :ref:`Use of add_inputs(...) | inputs(...) <new_manual.inputs>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
-####################################################
- at collate with ``add_inputs`` and ``inputs``
-####################################################
-
-.. |tasks_or_file_names| replace:: `tasks_or_file_names`
-.. _tasks_or_file_names: `decorators.collate_ex.tasks_or_file_names`_
-.. |extra_parameters| replace:: `extra_parameters`
-.. _extra_parameters: `decorators.collate_ex.extra_parameters`_
-.. |output_pattern| replace:: `output_pattern`
-.. _output_pattern: `decorators.collate_ex.output_pattern`_
-.. |input_pattern_or_glob| replace:: `input_pattern_or_glob`
-.. _input_pattern_or_glob: `decorators.collate_ex.input_pattern_or_glob`_
+.. |input| replace:: `input`
+.. _input: `decorators.collate_ex.input`_
+.. |extras| replace:: `extras`
+.. _extras: `decorators.collate_ex.extras`_
+.. |output| replace:: `output`
+.. _output: `decorators.collate_ex.output`_
+.. |filter| replace:: `filter`
+.. _filter: `decorators.collate_ex.filter`_
.. |matching_regex| replace:: `matching_regex`
.. _matching_regex: `decorators.collate_ex.matching_regex`_
.. |matching_formatter| replace:: `matching_formatter`
.. _matching_formatter: `decorators.collate_ex.matching_formatter`_
+.. |replace_inputs| replace:: `replace_inputs`
+.. _replace_inputs: `decorators.collate_ex.replace_inputs`_
+.. |add_inputs| replace:: `add_inputs`
+.. _add_inputs: `decorators.collate_ex.add_inputs`_
-************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************ [...]
-*@collate* ( |tasks_or_file_names|_, :ref:`regex<decorators.regex>`\ *(*\ |matching_regex|_\ *)* | :ref:`formatter<decorators.formatter>`\ *(*\ |matching_formatter|_\ *)*\, [:ref:`inputs<decorators.inputs>`\ *(*\ |input_pattern_or_glob|_\ *)* | :ref:`add_inputs<decorators.add_inputs>`\ *(*\ |input_pattern_or_glob|_\ *)*\] , |output_pattern|_, [|extra_parameters|_,...] )
-************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************ [...]
+################################################################################################################################################
+ at collate( |input|_, |filter|_, |replace_inputs|_ | |add_inputs|_, |output|_, [|extras|_,...] )
+################################################################################################################################################
**Purpose:**
- Groups / collates sets of input files, each into a separate summary.
+ Use |filter|_ to identify common sets of |input|_\s which are to be grouped or collated together:
- This variant of ``@collate`` allows additional inputs or dependencies to be added
- dynamically to the task.
-
- Output file names are determined from |tasks_or_file_names|_, i.e. from the output
- of up stream tasks, or a list of file names.
+ Each set of |input|_\ s which generate identical |output|_ and |extras|_ using the
+ :ref:`formatter<decorators.formatter>` or :ref:`regex<decorators.regex>` (regular expression)
+ filters are collated into one job.
- This variant of ``@collate`` allows input file names to be derived in the same way.
+ This variant of ``@collate`` allows additional inputs or dependencies to be added
+ dynamically to the task, with optional string substitution.
:ref:`add_inputs<decorators.add_inputs>` nests the the original input parameters in a list before adding additional dependencies.
:ref:`inputs<decorators.inputs>` replaces the original input parameters wholescale.
- Only out of date tasks (comparing input and output files) will be run
+ This is a **many to fewer** operation.
+
+ Only out of date jobs (comparing input and output files) will be re-run.
**Example of** :ref:`add_inputs<decorators.add_inputs>`
@@ -83,21 +94,23 @@
**Parameters:**
-.. _decorators.collate_ex.tasks_or_file_names:
+.. _decorators.collate_ex.input:
- * *tasks_or_file_names*
+ * **input** = *tasks_or_file_names*
can be a:
- #. Task / list of tasks (as in the example above).
+ #. Task / list of tasks.
File names are taken from the output of the specified task(s)
- #. (Nested) list of file name strings.
+ #. (Nested) list of file name strings (as in the example above).
File names containing ``*[]?`` will be expanded as a |glob|_.
E.g.:``"a.*" => "a.1", "a.2"``
+.. _decorators.collate_ex.filter:
+
.. _decorators.collate_ex.matching_regex:
- * *matching_regex*
+ * **filter** = *matching_regex*
is a python regular expression string, which must be wrapped in
a :ref:`regex<decorators.regex>` indicator object
See python `regular expression (re) <http://docs.python.org/library/re.html>`_
@@ -105,41 +118,41 @@
.. _decorators.collate_ex.matching_formatter:
- * *matching_formatter*
+ * **filter** = *matching_formatter*
a :ref:`formatter<decorators.formatter>` indicator object containing optionally
a python `regular expression (re) <http://docs.python.org/library/re.html>`_.
-.. _decorators.collate_ex.input_pattern_or_glob:
+.. _decorators.collate_ex.add_inputs:
+
+.. _decorators.collate_ex.replace_inputs:
+
+ * **add_inputs** = *add_inputs*\ (...) or **replace_inputs** = *inputs*\ (...)
+ Specifies the resulting |input|_\ (s) to each job.
- * *input_pattern*
- Specifies the resulting input(s) to each job.
- Must be wrapped in an :ref:`inputs<decorators.inputs>` or an :ref:`inputs<decorators.add_inputs>` indicator object.
+ Positional parameters must be disambiguated by wrapping the values in :ref:`inputs(...)<decorators.inputs>` or an :ref:`add_inputs(...)<decorators.add_inputs>`.
- Can be a:
+ Named parameters can be passed the values directly.
- #. Task / list of tasks (as in the example above).
+ Takes:
+
+ #. Task / list of tasks.
File names are taken from the output of the specified task(s)
#. (Nested) list of file name strings.
Strings will be subject to substitution.
File names containing ``*[]?`` will be expanded as a |glob|_.
- E.g.:``"a.*" => "a.1", "a.2"``
-
-
+ E.g. ``"a.*" => "a.1", "a.2"``
-.. _decorators.collate_ex.output_pattern:
+.. _decorators.collate_ex.output:
- * *output_pattern*
+ * **output** = *output*
Specifies the resulting output file name(s).
-.. _decorators.collate_ex.extra_parameters:
+.. _decorators.collate_ex.extras:
- * *extra_parameters*
+ * **extras** = *extras*
Any extra parameters are passed verbatim to the task function
- #. *outputs* and optional extra parameters are passed to the functions after string
- substitution in any strings. Non-string values are passed through unchanged.
- #. Each collate job consists of input files which are aggregated by string substitution
- to a single set of output / extra parameter matches
+ If you are using named parameters, these can be passed as a list, i.e. ``extras= [...]``
See :ref:`@collate <decorators.collate>` for more straightforward ways to use collate.
diff --git a/doc/decorators/combinations.rst b/doc/decorators/combinations.rst
index 65f9ec9..a4386e0 100644
--- a/doc/decorators/combinations.rst
+++ b/doc/decorators/combinations.rst
@@ -5,149 +5,218 @@
.. seealso::
+ * :ref:`@combinations <new_manual.combinations>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
-########################
- at combinations
-########################
-.. |tasks_or_file_names| replace:: `tasks_or_file_names`
-.. _tasks_or_file_names: `decorators.combinations.tasks_or_file_names`_
-.. |extra_parameters| replace:: `extra_parameters`
-.. _extra_parameters: `decorators.combinations.extra_parameters`_
-.. |output_pattern| replace:: `output_pattern`
-.. _output_pattern: `decorators.combinations.output_pattern`_
+.. |input| replace:: `input`
+.. _input: `decorators.combinations.input`_
+.. |tuple_size| replace:: `tuple_size`
+.. _tuple_size: `decorators.combinations.tuple_size`_
+.. |filter| replace:: `filter`
+.. _filter: `decorators.combinations.filter`_
+.. |extras| replace:: `extras`
+.. _extras: `decorators.combinations.extras`_
+.. |output| replace:: `output`
+.. _output: `decorators.combinations.output`_
.. |matching_formatter| replace:: `matching_formatter`
.. _matching_formatter: `decorators.combinations.matching_formatter`_
+################################################################################################################################################
+ at combinations( |input|_, |filter|_, |tuple_size|_, |output|_, [|extras|_,...] )
+################################################################################################################################################
-
-********************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
-*@combinations* ( |tasks_or_file_names|_, :ref:`formatter<decorators.formatter>`\ *(*\ |matching_formatter|_\ *)*\, |output_pattern|_, [|extra_parameters|_,...] )
-********************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
**Purpose:**
- Generates the **combinations**, between all the elements of a set of **Input** (e.g. **A B C D**),
- i.e. r-length tuples of *input* elements with no repeated elements (**A A**)
+ Generates the **combinations**, between all the elements of a set of |input|_ (e.g. **A B C D**),
+ i.e. r-length tuples of |input|_ elements with no repeated elements (*not* **A A**)
and where order of the tuples is irrelevant (either **A B** or **B A**, not both).
The effect is analogous to the python `itertools <http://docs.python.org/2/library/itertools.html#itertools.combinations>`__
function of the same name:
- .. code-block:: pycon
- :emphasize-lines: 2
+ .. code-block:: pycon
+ :emphasize-lines: 2
- >>> from itertools import combinations
- >>> # combinations('ABCD', 3) --> ABC ABD ACD BCD
- >>> [ "".join(a) for a in combinations("ABCD", 3)]
- ['ABC', 'ABD', 'ACD', 'BCD']
+ >>> from itertools import combinations
+ >>> # combinations('ABCD', 3) --> ABC ABD ACD BCD
+ >>> [ "".join(a) for a in combinations("ABCD", 3)]
+ ['ABC', 'ABD', 'ACD', 'BCD']
- Only out of date tasks (comparing input and output files) will be run
+ Only out of date tasks (comparing |input|_ and |output|_ files) will be run
- Output file names and strings in the extra parameters
- are determined from |tasks_or_file_names|_, i.e. from the output
- of up stream tasks, or a list of file names, after string replacement via
- :ref:`formatter<decorators.formatter>`.
+ |output|_ file names and strings in the extra parameters
+ are generated by string replacement via the :ref:`formatter()<decorators.formatter>` filter
+ from the |input|_. This can be, for example, a list of file names or the
+ |output|_ of up stream tasks.
+ .
+ The replacement strings require an extra level of nesting to refer to
+ parsed components.
- The replacement strings require an extra level of indirection to refer to
- parsed components:
+ #. The first level refers to which *set* in each tuple of |input|_.
+ #. The second level refers to which |input|_ file in any particular *set* of |input|_.
- #. The first level refers to which *set* in each tuple of inputs.
- #. The second level refers to which input file in any particular *set* of inputs.
+ This will be clear in the following example:
**Example**:
- Calculates the **@combinations** of **A,B,C,D** files
+ Calculate the **@combinations** of **A,B,C,D** files
+
+ If |input|_ is four pairs of file names
+
+ .. code-block:: python
+ :emphasize-lines: 1-6
+
+ input_files = [ [ 'A.1_start', 'A.2_start'], # 0
+ [ 'B.1_start', 'B.2_start'], # 1
+ [ 'C.1_start', 'C.2_start'], # 2
+ [ 'D.1_start', 'D.2_start'] ] # 3
+
+ The first job of:
+
+ .. code-block:: python
+
+ @combinations(input_files, formatter(), 3, ...)
+
+ Will be
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 1-6
+
+ # Three file pairs at a time
+ ['A.1_start', 'A.2_start'], # 0
+ # versus
+ ['B.1_start', 'B.2_start'], # 1
+ # versus
+ ['C.1_start', 'c.2_start'], # 2
- .. code-block:: python
- :emphasize-lines: 13,17,20,25,28-30
+ ..
+ python
- from ruffus import *
- from ruffus.combinatorics import *
+ First level of nesting:
+ .. code-block:: python
+ :emphasize-lines: 1-6
- # initial file pairs
- @originate([ ['A.1_start', 'A.2_start'],
- ['B.1_start', 'B.2_start'],
- ['C.1_start', 'C.2_start'],
- ['D.1_start', 'D.2_start']])
- def create_initial_files_ABCD(output_files):
- for output_file in output_files:
- with open(output_file, "w") as oo: pass
+ ['A.1_start', 'A.2_start'] # [0]
+ ['B.1_start', 'B.2_start'] # [1]
+ ['C.1_start', 'C.2_start'] # [2]
- # @combinations
- @combinations(create_initial_files_ABCD, # Input
- formatter(), # match input files
+ Second level of nesting:
+ .. code-block:: python
+ :emphasize-lines: 1-6
- # tuple of 3 at a time
- 3,
+ 'A.2_start' # [0][1]
+ 'B.2_start' # [1][1]
+ 'C.2_start' # [2][1]
- # Output Replacement string
- "{path[0][0]}/"
- "{basename[0][1]}_vs_"
- "{basename[1][1]}_vs_"
- "{basename[2][1]}.combinations",
+ Parse filename without suffix
+ .. code-block:: python
+ :emphasize-lines: 1-6
- # Extra parameter: path for 1st set of files, 1st file name
- "{path[0][0]}",
+ 'A' # {basename[0][1]}
+ 'B' # {basename[1][1]}
+ 'C' # {basename[2][1]}
- # Extra parameter
- ["{basename[0][0]}", # basename for 1st set of files, 1st file name
- "{basename[1][0]}", # 2nd
- "{basename[2][0]}", # 3rd
- ])
- def combinations_task(input_file, output_parameter, shared_path, basenames):
- print " - ".join(basenames)
+ Python code:
+ .. code-block:: python
+ :emphasize-lines: 13,17,20,25,28-30
- #
- # Run
- #
- pipeline_run(verbose=0)
+ from ruffus import *
+ from ruffus.combinatorics import *
+
+ # initial file pairs
+ @originate([ ['A.1_start', 'A.2_start'],
+ ['B.1_start', 'B.2_start'],
+ ['C.1_start', 'C.2_start'],
+ ['D.1_start', 'D.2_start']])
+ def create_initial_files_ABCD(output_files):
+ for output_file in output_files:
+ with open(output_file, "w") as oo: pass
+
+ # @combinations
+ @combinations(create_initial_files_ABCD, # Input
+ formatter(), # match input files
+
+ # tuple of 3 at a time
+ 3,
+
+ # Output Replacement string
+ "{path[0][0]}/"
+ "{basename[0][1]}_vs_"
+ "{basename[1][1]}_vs_"
+ "{basename[2][1]}.combinations",
+
+ # Extra parameter: path for 1st set of files, 1st file name
+ "{path[0][0]}",
+
+ # Extra parameter
+ ["{basename[0][0]}", # basename for 1st set of files, 1st file name
+ "{basename[1][0]}", # 2nd
+ "{basename[2][0]}", # 3rd
+ ])
+ def combinations_task(input_file, output_parameter, shared_path, basenames):
+ print " - ".join(basenames)
+
+
+ #
+ # Run
+ #
+ pipeline_run(verbose=0)
This results in:
- .. code-block:: pycon
+ .. code-block:: pycon
- >>> pipeline_run(verbose=0)
- A - B - C
- A - B - D
- A - C - D
- B - C - D
+ >>> pipeline_run(verbose=0)
+ A - B - C
+ A - B - D
+ A - C - D
+ B - C - D
**Parameters:**
-.. _decorators.combinations.tasks_or_file_names:
+.. _decorators.combinations.input:
- * *tasks_or_file_names*
+ * **input** = *tasks_or_file_names*
can be a:
- #. Task / list of tasks (as in the example above).
- File names are taken from the output of the specified task(s)
+ #. Task / list of tasks.
+ File names are taken from the |output|_ of the specified task(s)
#. (Nested) list of file name strings.
File names containing ``*[]?`` will be expanded as a |glob|_.
E.g.:``"a.*" => "a.1", "a.2"``
+.. _decorators.combinations.filter:
.. _decorators.combinations.matching_formatter:
- * *matching_formatter*
+ * **filter** = *formater(...)*
a :ref:`formatter<decorators.formatter>` indicator object containing optionally
a python `regular expression (re) <http://docs.python.org/library/re.html>`_.
+.. _decorators.combinations.tuple_size:
+
+ * **tuple_size** = *N*
+ Select N elements at a time.
+
+.. _decorators.combinations.output:
+
+ * **output** = *output*
+ Specifies the resulting output file name(s) after string substitution
-.. _decorators.combinations.output_pattern:
- * *output_pattern*
- Specifies the resulting output file name(s) after string
- substitution
+.. _decorators.combinations.extras:
+ * **extras** = *extras*
+ Any extra parameters are passed verbatim to the task function
-.. _decorators.combinations.extra_parameters:
+ If you are using named parameters, these can be passed as a list, i.e. ``extras= [...]``
- * *extra_parameters*
- Optional extra parameters are passed to the functions after string
- substitution
+ Any extra parameters are consumed by the task function and not forwarded further down the pipeline.
diff --git a/doc/decorators/combinations_with_replacement.rst b/doc/decorators/combinations_with_replacement.rst
index fbb0daf..40f3ee6 100644
--- a/doc/decorators/combinations_with_replacement.rst
+++ b/doc/decorators/combinations_with_replacement.rst
@@ -5,153 +5,215 @@
.. seealso::
+ * :ref:`@combinations_with_replacement <new_manual.combinations_with_replacement>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
-################################################
- at combinations_with_replacement
-################################################
-.. |tasks_or_file_names| replace:: `tasks_or_file_names`
-.. _tasks_or_file_names: `decorators.combinations_with_replacement.tasks_or_file_names`_
-.. |extra_parameters| replace:: `extra_parameters`
-.. _extra_parameters: `decorators.combinations_with_replacement.extra_parameters`_
-.. |output_pattern| replace:: `output_pattern`
-.. _output_pattern: `decorators.combinations_with_replacement.output_pattern`_
+.. |input| replace:: `input`
+.. _input: `decorators.combinations_with_replacement.input`_
+.. |tuple_size| replace:: `tuple_size`
+.. _tuple_size: `decorators.combinations_with_replacement.tuple_size`_
+.. |filter| replace:: `filter`
+.. _filter: `decorators.combinations_with_replacement.filter`_
+.. |extras| replace:: `extras`
+.. _extras: `decorators.combinations_with_replacement.extras`_
+.. |output| replace:: `output`
+.. _output: `decorators.combinations_with_replacement.output`_
.. |matching_formatter| replace:: `matching_formatter`
.. _matching_formatter: `decorators.combinations_with_replacement.matching_formatter`_
+################################################################################################################################################
+ at combinations_with_replacement( |input|_, |filter|_, |tuple_size|_, |output|_, [|extras|_,...] )
+################################################################################################################################################
-
-********************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
-*@combinations_with_replacement* ( |tasks_or_file_names|_, :ref:`formatter<decorators.formatter>`\ *(*\ |matching_formatter|_\ *)*\, |output_pattern|_, [|extra_parameters|_,...] )
-********************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
**Purpose:**
- Generates the **combinations_with_replacement**, between all the elements of a set of **Input** (e.g. **A B C D**),
- i.e. r-length tuples of *input* elements included repeated elements (**A A**)
+ Generates the **combinations_with_replacement**, between all the elements of a set of |input|_ (e.g. **A B C D**),
+ i.e. r-length tuples of |input|_ elements with no repeated elements (**A A**)
and where order of the tuples is irrelevant (either **A B** or **B A**, not both).
The effect is analogous to the python `itertools <http://docs.python.org/2/library/itertools.html#itertools.combinations_with_replacement>`__
function of the same name:
- .. code-block:: pycon
- :emphasize-lines: 2
+ .. code-block:: pycon
+ :emphasize-lines: 2
- >>> from itertools import combinations_with_replacement
- >>> # combinations_with_replacement('ABCD', 2) --> AA AB AC AD BB BC BD CC CD DD
- >>> [ "".join(a) for a in combinations_with_replacement('ABCD', 2)]
- ['AA', 'AB', 'AC', 'AD', 'BB', 'BC', 'BD', 'CC', 'CD', 'DD']
+ >>> from itertools import combinations_with_replacement
+ >>> # combinations_with_replacement('ABCD', 2) --> AA AB AC AD BB BC BD CC CD DD
+ >>> [ "".join(a) for a in combinations_with_replacement('ABCD', 2)]
+ ['AA', 'AB', 'AC', 'AD', 'BB', 'BC', 'BD', 'CC', 'CD', 'DD']
Only out of date tasks (comparing input and output files) will be run
- Output file names and strings in the extra parameters
- are determined from |tasks_or_file_names|_, i.e. from the output
- of up stream tasks, or a list of file names, after string replacement via
- :ref:`formatter<decorators.formatter>`.
+ |output|_ file names and strings in the extra parameters
+ are generated by string replacement via the :ref:`formatter()<decorators.formatter>` filter
+ from the |input|_. This can be, for example, a list of file names or the
+ |output|_ of up stream tasks.
+ .
+ The replacement strings require an extra level of nesting to refer to
+ parsed components.
- The replacement strings require an extra level of indirection to refer to
- parsed components:
+ #. The first level refers to which *set* in each tuple of |input|_.
+ #. The second level refers to which |input|_ file in any particular *set* of |input|_.
- #. The first level refers to which *set* in each tuple of inputs.
- #. The second level refers to which input file in any particular *set* of inputs.
+ This will be clear in the following example:
**Example**:
- Calculates the **@combinations_with_replacement** of **A,B,C,D** files
+ If |input|_ is four pairs of file names
+
+ .. code-block:: python
+ :emphasize-lines: 1-6
+
+ input_files = [ [ 'A.1_start', 'A.2_start'],
+ [ 'B.1_start', 'B.2_start'],
+ [ 'C.1_start', 'C.2_start'],
+ [ 'D.1_start', 'D.2_start'] ]
+
+ The first job of:
+
+ .. code-block:: python
+
+ @combinations_with_replacement(input_files, formatter(), 3, ...)
+
+ Will be
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 1-6
+
+ # Two file pairs at a time
+ ['A.1_start', 'A.2_start'], # 0
+ # versus itself
+ ['A.1_start', 'A.2_start'], # 1
+
+ ..
+ python
- .. code-block:: python
- :emphasize-lines: 13,17,20,25,28-30
+ First level of nesting:
+ .. code-block:: python
+ :emphasize-lines: 1-6
- from ruffus import *
- from ruffus.combinatorics import *
+ ['A.1_start', 'A.2_start'] # [0]
+ ['A.1_start', 'A.2_start'] # [1]
- # initial file pairs
- @originate([ ['A.1_start', 'A.2_start'],
- ['B.1_start', 'B.2_start'],
- ['C.1_start', 'C.2_start'],
- ['D.1_start', 'D.2_start']])
- def create_initial_files_ABCD(output_files):
- for output_file in output_files:
- with open(output_file, "w") as oo: pass
+ Second level of nesting:
+ .. code-block:: python
+ :emphasize-lines: 1-6
- # @combinations_with_replacement
- @combinations_with_replacement(create_initial_files_ABCD, # Input
- formatter(), # match input files
+ 'A.2_start' # [0][1]
+ 'A.2_start' # [1][1]
- # tuple of 2 at a time
- 2,
+ Parse filename without suffix
+ .. code-block:: python
+ :emphasize-lines: 1-6
- # Output Replacement string
- "{path[0][0]}/"
- "{basename[0][1]}_vs_"
- "{basename[1][1]}.combinations_with_replacement",
+ 'A' # {basename[0][1]}
+ 'A' # {basename[1][1]}
- # Extra parameter: path for 1st set of files, 1st file name
- "{path[0][0]}",
+ Python code:
- # Extra parameter
- ["{basename[0][0]}", # basename for 1st set of files, 1st file name
- "{basename[1][0]}", # 2rd
- ])
- def combinations_with_replacement_task(input_file, output_parameter, shared_path, basenames):
- print " - ".join(basenames)
+ .. code-block:: python
+ :emphasize-lines: 13,17,20,25,28-30
+ from ruffus import *
+ from ruffus.combinatorics import *
- #
- # Run
- #
- pipeline_run(verbose=0)
+ # initial file pairs
+ @originate([ ['A.1_start', 'A.2_start'],
+ ['B.1_start', 'B.2_start'],
+ ['C.1_start', 'C.2_start'],
+ ['D.1_start', 'D.2_start']])
+ def create_initial_files_ABCD(output_files):
+ for output_file in output_files:
+ with open(output_file, "w") as oo: pass
+
+ # @combinations_with_replacement
+ @combinations_with_replacement(create_initial_files_ABCD, # Input
+ formatter(), # match input files
+
+ # tuple of 2 at a time
+ 2,
+
+ # Output Replacement string
+ "{path[0][0]}/"
+ "{basename[0][1]}_vs_"
+ "{basename[1][1]}.combinations_with_replacement",
+
+ # Extra parameter: path for 1st set of files, 1st file name
+ "{path[0][0]}",
+
+ # Extra parameter
+ ["{basename[0][0]}", # basename for 1st set of files, 1st file name
+ "{basename[1][0]}", # 2rd
+ ])
+ def combinations_with_replacement_task(input_file, output_parameter, shared_path, basenames):
+ print " - ".join(basenames)
+
+
+ #
+ # Run
+ #
+ pipeline_run(verbose=0)
This results in:
- .. code-block:: pycon
+ .. code-block:: pycon
- >>> pipeline_run(verbose=0)
- A - A
- A - B
- A - C
- A - D
- B - B
- B - C
- B - D
- C - C
- C - D
- D - D
+ >>> pipeline_run(verbose=0)
+ A - A
+ A - B
+ A - C
+ A - D
+ B - B
+ B - C
+ B - D
+ C - C
+ C - D
+ D - D
**Parameters:**
-.. _decorators.combinations_with_replacement.tasks_or_file_names:
+.. _decorators.combinations_with_replacement.input:
- * *tasks_or_file_names*
+ * **input** = *tasks_or_file_names*
can be a:
- #. Task / list of tasks (as in the example above).
- File names are taken from the output of the specified task(s)
+ #. Task / list of tasks.
+ File names are taken from the |output|_ of the specified task(s)
#. (Nested) list of file name strings.
File names containing ``*[]?`` will be expanded as a |glob|_.
E.g.:``"a.*" => "a.1", "a.2"``
+.. _decorators.combinations_with_replacement.filter:
.. _decorators.combinations_with_replacement.matching_formatter:
- * *matching_formatter*
+ * **filter** = *formater(...)*
a :ref:`formatter<decorators.formatter>` indicator object containing optionally
a python `regular expression (re) <http://docs.python.org/library/re.html>`_.
+.. _decorators.combinations_with_replacement.tuple_size:
+
+ * **tuple_size** = *N*
+ Select N elements at a time.
+
+.. _decorators.combinations_with_replacement.output:
+
+ * **output** = *output*
+ Specifies the resulting output file name(s) after string substitution
-.. _decorators.combinations_with_replacement.output_pattern:
- * *output_pattern*
- Specifies the resulting output file name(s) after string
- substitution
+.. _decorators.combinations_with_replacement.extras:
+ * **extras** = *extras*
+ Any extra parameters are passed verbatim to the task function
-.. _decorators.combinations_with_replacement.extra_parameters:
+ If you are using named parameters, these can be passed as a list, i.e. ``extras= [...]``
- * *extra_parameters*
- Optional extra parameters are passed to the functions after string
- substitution
+ Any extra parameters are consumed by the task function and not forwarded further down the pipeline.
diff --git a/doc/decorators/files.rst b/doc/decorators/files.rst
index 8f73ba5..7e03c97 100644
--- a/doc/decorators/files.rst
+++ b/doc/decorators/files.rst
@@ -5,9 +5,9 @@
.. seealso::
+ * :ref:`@files (deprecated) <new_manual.deprecated_files>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
-
.. |input| replace:: `input`
.. _input: `decorators.files.input`_
.. |input1| replace:: `input1`
@@ -146,8 +146,8 @@
**Checking if jobs are up to date:**
#. Strings in ``input`` and ``output`` (including in nested sequences) are interpreted as file names and
used to check if jobs are up-to-date.
- #. In the absence of input files (e.g. ``input == None``), the job will run if any output file is missing.
- #. In the absence of output files (e.g. ``output == None``), the job will always run.
+ #. In the absence of input files (e.g. ``input is None``), the job will run if any output file is missing.
+ #. In the absence of output files (e.g. ``output is None``), the job will always run.
#. If any of the output files is missing, the job will run.
#. If any of the input files is missing when the job is run, a
``MissingInputFileError`` exception will be raised.
diff --git a/doc/decorators/files_ex.rst b/doc/decorators/files_ex.rst
index d200d85..c15e66c 100644
--- a/doc/decorators/files_ex.rst
+++ b/doc/decorators/files_ex.rst
@@ -5,9 +5,9 @@
.. seealso::
+ * :ref:`@files <new_manual.on_the_fly>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
-
.. |custom_function| replace:: `custom_function`
.. _custom_function: `decorators.files.custom_function`_
diff --git a/doc/decorators/follows.rst b/doc/decorators/follows.rst
index 58e6d5c..68e677f 100644
--- a/doc/decorators/follows.rst
+++ b/doc/decorators/follows.rst
@@ -5,8 +5,8 @@
.. seealso::
+ * :ref:`@follows <new_manual.follows>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
- * More on @follows in the ``Ruffus`` :ref:`Manual <new_manual.follows>`
.. note::
diff --git a/doc/decorators/graphviz.rst b/doc/decorators/graphviz.rst
index ebda4df..e939f81 100644
--- a/doc/decorators/graphviz.rst
+++ b/doc/decorators/graphviz.rst
@@ -5,6 +5,7 @@
.. seealso::
+ * :ref:`@graphviz <new_manual.graphviz>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
########################
diff --git a/doc/decorators/indicator_objects.rst b/doc/decorators/indicator_objects.rst
index e41cc4c..e2389bd 100644
--- a/doc/decorators/indicator_objects.rst
+++ b/doc/decorators/indicator_objects.rst
@@ -2,7 +2,10 @@
.. seealso::
- :ref:`Decorators <decorators>`
+ * :ref:`Decorators <decorators>`
+ * :ref:`suffix(...) <new_manual.suffix>` in the **Ruffus** Manual
+ * :ref:`regex(...) <new_manual.regex>` in the **Ruffus** Manual
+ * :ref:`formatter(...) <new_manual.formatter>` in the **Ruffus** Manual
.. index::
single: Indicator Object (Disambiguating parameters)
diff --git a/doc/decorators/jobs_limit.rst b/doc/decorators/jobs_limit.rst
index 1e0ee41..a799dc8 100644
--- a/doc/decorators/jobs_limit.rst
+++ b/doc/decorators/jobs_limit.rst
@@ -5,9 +5,9 @@
.. seealso::
+ * :ref:`@jobs_limit <new_manual.jobs_limit>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
-
########################
@jobs_limit
########################
diff --git a/doc/decorators/merge.rst b/doc/decorators/merge.rst
index bd85ee8..199619b 100644
--- a/doc/decorators/merge.rst
+++ b/doc/decorators/merge.rst
@@ -5,26 +5,24 @@
.. seealso::
+ * :ref:`@merge <new_manual.merge>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
-.. |tasks_or_file_names| replace:: `tasks_or_file_names`
-.. _tasks_or_file_names: `decorators.merge.tasks_or_file_names`_
-.. |extra_parameters| replace:: `extra_parameters`
-.. _extra_parameters: `decorators.merge.extra_parameters`_
-.. |output_file| replace:: `output_file`
-.. _output_file: `decorators.merge.output_file`_
+.. |input| replace:: `input`
+.. _input: `decorators.merge.input`_
+.. |extras| replace:: `extras`
+.. _extras: `decorators.merge.extras`_
+.. |output| replace:: `output`
+.. _output: `decorators.merge.output`_
-########################
- at merge
-########################
+########################################################################
+ at merge ( |input|_, |output|_, [|extras|_,...] )
+########################################################################
-************************************************************************************
-*@merge* ( |tasks_or_file_names|_, |output_file|_, [|extra_parameters|_,...] )
-************************************************************************************
**Purpose:**
- Merges multiple input files into a single output.
+ Merges multiple |input|_ into a single |output|_.
- Only out of date tasks (comparing input and output files) will be run
+ Only out of date tasks (comparing |input|_ and |output|_ files) will be run
**Example**::
@@ -35,27 +33,31 @@
**Parameters:**
-.. _decorators.merge.tasks_or_file_names:
+.. _decorators.merge.input:
- * *tasks_or_file_names*
+ * **input** = *tasks_or_file_names*
can be a:
- #. Task / list of tasks (as in the example above).
+ #. Task / list of tasks.
File names are taken from the output of the specified task(s)
#. (Nested) list of file name strings.
File names containing ``*[]?`` will be expanded as a |glob|_.
E.g.:``"a.*" => "a.1", "a.2"``
-.. _decorators.merge.output_file:
+.. _decorators.merge.output:
- * *output_file*
+ * **output** = *output*
Specifies the resulting output file name(s).
-.. _decorators.merge.extra_parameters:
+.. _decorators.merge.extras:
- * *extra_parameters, ...*
- Any optional extra parameters are passed verbatim to the task function
+ * **extras** = *extras*
+ Any extra parameters are passed verbatim to the task function
+
+ If you are using named parameters, these can be passed as a list, i.e. ``extras= [...]``
+
+ Any extra parameters are consumed by the task function and not forwarded further down the pipeline.
diff --git a/doc/decorators/mkdir.rst b/doc/decorators/mkdir.rst
index b516b1a..e87896f 100644
--- a/doc/decorators/mkdir.rst
+++ b/doc/decorators/mkdir.rst
@@ -5,18 +5,16 @@
.. seealso::
- * :ref:`Decorators <decorators>` for more decorators
- * More on @mkdir in the ``Ruffus`` :ref:`Manual <new_manual.mkdir>`
+ * :ref:`@mkdir <new_manual.mkdir>` in the **Ruffus** Manual
* :ref:`@follows(mkdir("dir")) <decorators.follows>` specifies the creation of a *single* directory as a task pre-requisite.
+ * :ref:`Decorators <decorators>` for more decorators
-########################
- at mkdir
-########################
-
-.. |tasks_or_file_names| replace:: `tasks_or_file_names`
-.. _tasks_or_file_names: `decorators.mkdir.tasks_or_file_names`_
-.. |output_pattern| replace:: `output_pattern`
-.. _output_pattern: `decorators.mkdir.output_pattern`_
+.. |input| replace:: `input`
+.. _input: `decorators.mkdir.input`_
+.. |output| replace:: `output`
+.. _output: `decorators.mkdir.output`_
+.. |filter| replace:: `filter`
+.. _filter: `decorators.mkdir.filter`_
.. |matching_regex| replace:: `matching_regex`
.. _matching_regex: `decorators.mkdir.matching_regex`_
.. |matching_formatter| replace:: `matching_formatter`
@@ -24,9 +22,9 @@
.. |suffix_string| replace:: `suffix_string`
.. _suffix_string: `decorators.mkdir.suffix_string`_
-******************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
-*@mkdir* ( |tasks_or_file_names|_, :ref:`suffix<decorators.suffix>`\ *(*\ |suffix_string|_\ *)*\ | :ref:`regex<decorators.regex>`\ *(*\ |matching_regex|_\ *)* | :ref:`formatter<decorators.formatter>`\ *(*\ |matching_formatter|_\ *)*\, |output_pattern|_)
-******************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
+########################################################################
+ at mkdir( |input|_, |filter|_, |output|_ )
+########################################################################
**Purpose:**
* Prepares directories to receive *Output* files
@@ -128,38 +126,31 @@
Job = [B.start -> [processed.txt, tmp.processed.txt]] completed
Completed Task = create_files_with_mkdir
-
-
- **Escaping regular expression patterns**
-
- A string like ``universal.h`` in ``add_inputs`` will added *as is*.
- ``r"\1.h"``, however, performs suffix substitution, with the special form ``r"\1"`` matching everything up to the suffix.
- Remember to 'escape' ``r"\1"`` otherwise Ruffus will complain and throw an Exception to remind you.
- The most convenient way is to use a python "raw" string.
-
**Parameters:**
-.. _decorators.mkdir.tasks_or_file_names:
+.. _decorators.mkdir.input:
- * *tasks_or_file_names*
+ * **input** = *tasks_or_file_names*
can be a:
#. Task / list of tasks (as in the example above).
- File names are taken from the output of the specified task(s)
+ File names are taken from the |output|_ of the specified task(s)
#. (Nested) list of file name strings.
File names containing ``*[]?`` will be expanded as a |glob|_.
E.g.:``"a.*" => "a.1", "a.2"``
+.. _decorators.mkdir.filter:
+
.. _decorators.mkdir.suffix_string:
- * *suffix_string*
+ * **filter** = *suffix(suffix_string)*
must be wrapped in a :ref:`suffix<decorators.suffix>` indicator object.
- The end of each input file name which matches ``suffix_string`` will be replaced by ``output_pattern``.
+ The end of each |input|_ file name which matches ``suffix_string`` will be replaced by |output|_.
Input file names which do not match suffix_string will be ignored
- The non-suffix part of the match can be referred to using the ``"\1"`` pattern. This
+ The non-suffix part of the match can be referred to using the ``r"\1"`` pattern. This
can be useful for putting the output in different directory, for example::
@@ -200,21 +191,21 @@
.. _decorators.mkdir.matching_regex:
- * *matching_regex*
+ * **filter** = *regex(matching_regex)*
is a python regular expression string, which must be wrapped in
a :ref:`regex<decorators.regex>`\ indicator object
See python `regular expression (re) <http://docs.python.org/library/re.html>`_
documentation for details of regular expression syntax
- Each output file name is created using regular expression substitution with ``output_pattern``
+ Each output file name is created using regular expression substitution with ``output``
.. _decorators.mkdir.matching_formatter:
- * *matching_formatter*
+ * **filter** = *formatter(...)*
a :ref:`formatter<decorators.formatter>` indicator object containing optionally
a python `regular expression (re) <http://docs.python.org/library/re.html>`_.
-.. _decorators.mkdir.output_pattern:
+.. _decorators.mkdir.output:
- * *output_pattern*
- Specifies the resulting output file name(s).
+ * **output** = *output*
+ Specifies the directories to be created after string substitution
diff --git a/doc/decorators/originate.rst b/doc/decorators/originate.rst
index 513a285..c43fab0 100644
--- a/doc/decorators/originate.rst
+++ b/doc/decorators/originate.rst
@@ -5,30 +5,28 @@
.. seealso::
+ * :ref:`@originate <new_manual.originate>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
-########################
- at originate
-########################
+################################################
+*@originate* ( |output|_, [|extras|_,...] )
+################################################
-.. |output_files| replace:: `output_files`
-.. _output_files: `decorators.originate.output_files`_
-.. |extra_parameters| replace:: `extra_parameters`
-.. _extra_parameters: `decorators.originate.extra_parameters`_
+.. |output| replace:: `output`
+.. _output: `decorators.originate.output`_
+.. |extras| replace:: `extras`
+.. _extras: `decorators.originate.extras`_
-***********************************************************************************************************************************************************
-*@originate* ( |output_files|_, [|extra_parameters|_,...] )
-***********************************************************************************************************************************************************
**Purpose:**
* Creates (originates) a set of starting file without dependencies from scratch (*ex nihilo*!)
* Only called to create files which do not exist.
- * Invoked onces (a job created) per item in the ``output_files`` list.
+ * Invoked onces (a job created) per item in the |output|_ list.
.. note::
- The first argument for the task function is the *Output*. There is by definition no
- *Input* for ``@originate``
+ The first argument for the task function is the |output|_. There is by definition no
+ *input* for ``@originate``
**Example**:
@@ -65,15 +63,19 @@
**Parameters:**
-.. _decorators.originate.output_files:
+.. _decorators.originate.output:
- * *output_files*
+ * **output** = *output*
* Can be a single file name or a list of files
- * Each item in the list is treated as the *Output* of a separate job
+ * Each item in the list is treated as the |output|_ of a separate job
-.. _decorators.originate.extra_parameters:
+.. _decorators.originate.extras:
- * *extra_parameters*
- Any extra parameters are passed verbatim to the task function
+ * **extras** = *extras*
+ Any extra parameters are passed verbatim to the task function
+
+ If you are using named parameters, these can be passed as a list, i.e. ``extras= [...]``
+
+ Any extra parameters are consumed by the task function and not forwarded further down the pipeline.
diff --git a/doc/decorators/parallel.rst b/doc/decorators/parallel.rst
index f16cd48..269af33 100644
--- a/doc/decorators/parallel.rst
+++ b/doc/decorators/parallel.rst
@@ -5,9 +5,9 @@
.. seealso::
+ * :ref:`@parallel <new_manual.parallel>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
-
########################
@parallel
########################
diff --git a/doc/decorators/permutations.rst b/doc/decorators/permutations.rst
index 1ef39a6..9d231c9 100644
--- a/doc/decorators/permutations.rst
+++ b/doc/decorators/permutations.rst
@@ -5,29 +5,28 @@
.. seealso::
+ * :ref:`@permutations <new_manual.permutations>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
-########################
- at permutations
-########################
-
-.. |tasks_or_file_names| replace:: `tasks_or_file_names`
-.. _tasks_or_file_names: `decorators.permutations.tasks_or_file_names`_
-.. |extra_parameters| replace:: `extra_parameters`
-.. _extra_parameters: `decorators.permutations.extra_parameters`_
-.. |output_pattern| replace:: `output_pattern`
-.. _output_pattern: `decorators.permutations.output_pattern`_
+.. |input| replace:: `input`
+.. _input: `decorators.permutations.input`_
+.. |tuple_size| replace:: `tuple_size`
+.. _tuple_size: `decorators.permutations.tuple_size`_
+.. |filter| replace:: `filter`
+.. _filter: `decorators.permutations.filter`_
+.. |extras| replace:: `extras`
+.. _extras: `decorators.permutations.extras`_
+.. |output| replace:: `output`
+.. _output: `decorators.permutations.output`_
.. |matching_formatter| replace:: `matching_formatter`
.. _matching_formatter: `decorators.permutations.matching_formatter`_
-
-
-********************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
-*@permutations* ( |tasks_or_file_names|_, :ref:`formatter<decorators.formatter>`\ *(*\ |matching_formatter|_\ *)*\, |output_pattern|_, [|extra_parameters|_,...] )
-********************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
+################################################################################################################################################
+ at permutations( |input|_, |filter|_, |tuple_size|_, |output|_, [|extras|_,...] )
+################################################################################################################################################
**Purpose:**
- Generates the **permutations**, between all the elements of a set of **Input**
+ Generates the **permutations**, between all the elements of a set of |input|_ (e.g. **A B C D**),
The effect is analogous to the python `itertools <http://docs.python.org/2/library/itertools.html#itertools.permutations>`__
function of the same name:
@@ -42,117 +41,180 @@
Only out of date tasks (comparing input and output files) will be run
- Output file names and strings in the extra parameters
- are determined from |tasks_or_file_names|_, i.e. from the output
- of up stream tasks, or a list of file names, after string replacement via
- :ref:`formatter<decorators.formatter>`.
+ |output|_ file names and strings in the extra parameters
+ are generated by string replacement via the :ref:`formatter()<decorators.formatter>` filter
+ from the |input|_. This can be, for example, a list of file names or the
+ |output|_ of up stream tasks.
+ .
+ The replacement strings require an extra level of nesting to refer to
+ parsed components.
- The replacement strings require an extra level of indirection to refer to
- parsed components:
+ #. The first level refers to which *set* in each tuple of |input|_.
+ #. The second level refers to which |input|_ file in any particular *set* of |input|_.
- #. The first level refers to which *set* in each tuple of inputs.
- #. The second level refers to which input file in any particular *set* of inputs.
+ This will be clear in the following example:
**Example**:
- Calculates the **@permutations** of **A,B,C,D** files
+ Calculate the **@permutations** of **A,B,C,D** files
+
+ If |input|_ is four pairs of file names
+
+ .. code-block:: python
+ :emphasize-lines: 1-6
+
+ input_files = [ [ 'A.1_start', 'A.2_start'], # 0
+ [ 'B.1_start', 'B.2_start'], # 1
+ [ 'C.1_start', 'C.2_start'], # 2
+ [ 'D.1_start', 'D.2_start'] ] # 3
+
+ The first job of:
+
+ .. code-block:: python
+
+ @permutations(input_files, formatter(), 2, ...)
+
+ Will be
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 1-6
+
+ # Two file pairs at a time
+ ['A.1_start', 'A.2_start'], # 0
+ # versus
+ ['B.1_start', 'B.2_start'], # 1
- .. code-block:: python
- :emphasize-lines: 13,17,20,25,28-30
+ ..
+ python
- from ruffus import *
- from ruffus.combinatorics import *
+ First level of nesting:
+ .. code-block:: python
+ :emphasize-lines: 1-6
- # initial file pairs
- @originate([ ['A.1_start', 'A.2_start'],
- ['B.1_start', 'B.2_start'],
- ['C.1_start', 'C.2_start'],
- ['D.1_start', 'D.2_start']])
- def create_initial_files_ABCD(output_files):
- for output_file in output_files:
- with open(output_file, "w") as oo: pass
+ ['A.1_start', 'A.2_start'] # [0]
+ ['B.1_start', 'B.2_start'] # [1]
- # @permutations
- @permutations(create_initial_files_ABCD, # Input
- formatter(), # match input files
+ Second level of nesting:
+ .. code-block:: python
+ :emphasize-lines: 1-6
- # tuple of 2 at a time
- 2,
+ 'A.2_start' # [0][1]
+ 'B.2_start' # [1][1]
- # Output Replacement string
- "{path[0][0]}/"
- "{basename[0][1]}_vs_"
- "{basename[1][1]}.permutations",
+ Parse filename without suffix
+ .. code-block:: python
+ :emphasize-lines: 1-6
- # Extra parameter: path for 1st set of files, 1st file name
- "{path[0][0]}",
+ 'A' # {basename[0][1]}
+ 'B' # {basename[1][1]}
- # Extra parameter
- ["{basename[0][0]}", # basename for 1st set of files, 1st file name
- "{basename[1][0]}", # 2nd
- ])
- def permutations_task(input_file, output_parameter, shared_path, basenames):
- print " - ".join(basenames)
+ Python code:
+ .. code-block:: python
+ :emphasize-lines: 13,17,20,25,28-30
- #
- # Run
- #
- pipeline_run(verbose=0)
+ from ruffus import *
+ from ruffus.combinatorics import *
+
+ # initial file pairs
+ @originate([ ['A.1_start', 'A.2_start'],
+ ['B.1_start', 'B.2_start'],
+ ['C.1_start', 'C.2_start'],
+ ['D.1_start', 'D.2_start']])
+ def create_initial_files_ABCD(output_files):
+ for output_file in output_files:
+ with open(output_file, "w") as oo: pass
+
+ # @permutations
+ @permutations(create_initial_files_ABCD, # Input
+ formatter(), # match input files
+
+ # tuple of 2 at a time
+ 2,
+
+ # Output Replacement string
+ "{path[0][0]}/"
+ "{basename[0][1]}_vs_"
+ "{basename[1][1]}.permutations",
+
+ # Extra parameter: path for 1st set of files, 1st file name
+ "{path[0][0]}",
+
+ # Extra parameter
+ ["{basename[0][0]}", # basename for 1st set of files, 1st file name
+ "{basename[1][0]}", # 2nd
+ ])
+ def permutations_task(input_file, output_parameter, shared_path, basenames):
+ print " - ".join(basenames)
+
+
+ #
+ # Run
+ #
+ pipeline_run(verbose=0)
This results in:
- .. code-block:: pycon
+ .. code-block:: pycon
- >>> pipeline_run(verbose=0)
+ >>> pipeline_run(verbose=0)
- A - B
- A - C
- A - D
- B - A
- B - C
- B - D
- C - A
- C - B
- C - D
- D - A
- D - B
- D - C
+ A - B
+ A - C
+ A - D
+ B - A
+ B - C
+ B - D
+ C - A
+ C - B
+ C - D
+ D - A
+ D - B
+ D - C
**Parameters:**
-.. _decorators.permutations.tasks_or_file_names:
+.. _decorators.permutations.input:
- * *tasks_or_file_names*
+ * **input** = *tasks_or_file_names*
can be a:
- #. Task / list of tasks (as in the example above).
- File names are taken from the output of the specified task(s)
+ #. Task / list of tasks.
+ File names are taken from the |output|_ of the specified task(s)
#. (Nested) list of file name strings.
File names containing ``*[]?`` will be expanded as a |glob|_.
E.g.:``"a.*" => "a.1", "a.2"``
+.. _decorators.permutations.filter:
.. _decorators.permutations.matching_formatter:
- * *matching_formatter*
+ * **filter** = *formater(...)*
a :ref:`formatter<decorators.formatter>` indicator object containing optionally
a python `regular expression (re) <http://docs.python.org/library/re.html>`_.
+.. _decorators.permutations.tuple_size:
+
+ * **tuple_size** = *N*
+ Select N elements at a time.
+
+.. _decorators.permutations.output:
+
+ * **output** = *output*
+ Specifies the resulting output file name(s) after string substitution
-.. _decorators.permutations.output_pattern:
- * *output_pattern*
- Specifies the resulting output file name(s) after string
- substitution
+.. _decorators.permutations.extras:
+ * **extras** = *extras*
+ Any extra parameters are passed verbatim to the task function
-.. _decorators.permutations.extra_parameters:
+ If you are using named parameters, these can be passed as a list, i.e. ``extras= [...]``
- * *extra_parameters*
- Optional extra parameters are passed to the functions after string
- substitution
+ Any extra parameters are consumed by the task function and not forwarded further down the pipeline.
diff --git a/doc/decorators/posttask.rst b/doc/decorators/posttask.rst
index e0a6a2f..548cb3c 100644
--- a/doc/decorators/posttask.rst
+++ b/doc/decorators/posttask.rst
@@ -5,6 +5,7 @@
.. seealso::
+ * :ref:`@posttask <new_manual.posttask>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
.. |function| replace:: `function`
diff --git a/doc/decorators/product.rst b/doc/decorators/product.rst
index 5c77af4..5cab39f 100644
--- a/doc/decorators/product.rst
+++ b/doc/decorators/product.rst
@@ -5,29 +5,32 @@
.. seealso::
+ * :ref:`@product <new_manual.product>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
-########################
- at product
-########################
-
-.. |tasks_or_file_names| replace:: `tasks_or_file_names`
-.. _tasks_or_file_names: `decorators.product.tasks_or_file_names`_
-.. |extra_parameters| replace:: `extra_parameters`
-.. _extra_parameters: `decorators.product.extra_parameters`_
-.. |output_pattern| replace:: `output_pattern`
-.. _output_pattern: `decorators.product.output_pattern`_
+.. |input| replace:: `input`
+.. _input: `decorators.product.input`_
+.. |filter| replace:: `filter`
+.. _filter: `decorators.product.filter`_
+.. |input2| replace:: `input2`
+.. _input2: `decorators.product.input2`_
+.. |filter2| replace:: `filter2`
+.. _filter2: `decorators.product.filter2`_
+.. |extras| replace:: `extras`
+.. _extras: `decorators.product.extras`_
+.. |output| replace:: `output`
+.. _output: `decorators.product.output`_
.. |matching_formatter| replace:: `matching_formatter`
.. _matching_formatter: `decorators.product.matching_formatter`_
+################################################################################################################################################
+ at product( |input|_, |filter|_, [|input2|_, |filter2|_, ...], |output|_, [|extras|_,...] )
+################################################################################################################################################
-********************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
-*@product* ( |tasks_or_file_names|_, :ref:`formatter<decorators.formatter>`\ *(*\ |matching_formatter|_\ *)*\, [|tasks_or_file_names|_, :ref:`formatter<decorators.formatter>`\ *(*\ |matching_formatter|_\ *)*\, ... ], |output_pattern|_, [|extra_parameters|_,...] )
-********************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
**Purpose:**
- Generates the Cartesian **product**, i.e. all vs all comparisons, between sets of input files.
+ Generates the Cartesian **product**, i.e. all vs all comparisons, between multiple sets of |input|_ (e.g. **A B C D**, and **X Y Z**),
The effect is analogous to the python `itertools <http://docs.python.org/2/library/itertools.html#itertools.product>`__
function of the same name, i.e. a nested for loop.
@@ -42,151 +45,231 @@
Only out of date tasks (comparing input and output files) will be run
- Output file names and strings in the extra parameters
- are determined from |tasks_or_file_names|_, i.e. from the output
- of up stream tasks, or a list of file names, after string replacement via
- :ref:`formatter<decorators.formatter>`.
-
- The replacement strings require an extra level of indirection to refer to
- parsed components:
+ |output|_ file names and strings in the extra parameters
+ are generated by string replacement via the :ref:`formatter()<decorators.formatter>` filter
+ from the |input|_. This can be, for example, a list of file names or the
+ |output|_ of up stream tasks.
+ .
+ The replacement strings require an extra level of nesting to refer to
+ parsed components.
- #. The first level refers to which *set* of inputs (e.g. **A,B** or **P,Q** or **X,Y**
- in the following example.)
- #. The second level refers to which input file in any particular *set* of inputs.
+ #. The first level refers to which *set* in each tuple of |input|_.
+ #. The second level refers to which |input|_ file in any particular *set* of |input|_.
- For example, ``'{basename[2][0]}'`` is the `basename <http://docs.python.org/2/library/os.path.html#os.path.basename>`__ for
- * the third set of inputs (**X,Y**) and
- * the first file name string in each **Input** of that set (``"x.1_start"`` and ``"y.1_start"``)
+ This will be clear in the following example:
**Example**:
Calculates the **@product** of **A,B** and **P,Q** and **X, Y** files
- .. code-block:: python
- :emphasize-lines: 4,17,19,22,25,27,28,29,30,32,34,35,36
+ If |input|_ is three *sets* of file names
+
+ .. code-block:: python
+ :emphasize-lines: 1-10
+
+ set1 = [ 'a.start', # 0
+ 'b.start'])
+
+ set2 = [ 'p.start', # 1
+ 'q.start'])
+
+ set3 = [ ['x.1_start', 'x.2_start'], # 2
+ ['y.1_start', 'y.2_start'] ]
+
+ The first job of:
+
+ .. code-block:: python
+
+ @product( input = set1, filter = formatter(),
+ input2 = set2, filter2 = formatter(),
+ input3 = set2, filter3 = formatter(),
+ ...)
+
+ Will be
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 1-6
+
+ # One from each set
+ ['a.start']
+ # versus
+ ['p.start']
+ # versus
+ ['x.1_start', 'x.2_start'],
+ ..
+ python
+
+ First level of nesting (one list of files from each set):
+ .. code-block:: python
+ :emphasize-lines: 1-6
+
+ ['a.start'] # [0]
+ ['p.start'] # [1]
+ ['x.1_start', 'x.2_start'], # [2]
+
+ Second level of nesting (one file):
+ .. code-block:: python
+ :emphasize-lines: 1-6
- from ruffus import *
- from ruffus.combinatorics import *
+ 'a.start' # [0][0]
+ 'p.start' # [1][0]
+ 'x.1_start' # [2][0]
- # Three sets of initial files
- @originate([ 'a.start', 'b.start'])
- def create_initial_files_ab(output_file):
- with open(output_file, "w") as oo: pass
+ Parse filename without suffix
+ .. code-block:: python
+ :emphasize-lines: 1-6
- @originate([ 'p.start', 'q.start'])
- def create_initial_files_pq(output_file):
- with open(output_file, "w") as oo: pass
+ 'a' # {basename[0][0]}
+ 'p' # {basename[1][0]}
+ 'x' # {basename[2][0]}
- @originate([ ['x.1_start', 'x.2_start'],
- ['y.1_start', 'y.2_start'] ])
- def create_initial_files_xy(output_file):
- with open(output_file, "w") as oo: pass
+ Python code:
- # @product
- @product( create_initial_files_ab, # Input
- formatter("(.start)$"), # match input file set # 1
- create_initial_files_pq, # Input
- formatter("(.start)$"), # match input file set # 2
+ .. code-block:: python
+ :emphasize-lines: 4,17,19,22,25,27,28,29,30,32,34,35,36
- create_initial_files_xy, # Input
- formatter("(.start)$"), # match input file set # 3
+ from ruffus import *
+ from ruffus.combinatorics import *
- "{path[0][0]}/" # Output Replacement string
- "{basename[0][0]}_vs_" #
- "{basename[1][0]}_vs_" #
- "{basename[2][0]}.product", #
+ # Three sets of initial files
+ @originate([ 'a.start', 'b.start'])
+ def create_initial_files_ab(output_file):
+ with open(output_file, "w") as oo: pass
- "{path[0][0]}", # Extra parameter: path for 1st set of files, 1st file name
+ @originate([ 'p.start', 'q.start'])
+ def create_initial_files_pq(output_file):
+ with open(output_file, "w") as oo: pass
- ["{basename[0][0]}", # Extra parameter: basename for 1st set of files, 1st file name
- "{basename[1][0]}", # 2nd
- "{basename[2][0]}", # 3rd
- ])
- def product_task(input_file, output_parameter, shared_path, basenames):
- print "# basenames = ", " ".join(basenames)
- print "input_parameter = ", input_file
- print "output_parameter = ", output_parameter, "\n"
+ @originate([ ['x.1_start', 'x.2_start'],
+ ['y.1_start', 'y.2_start'] ])
+ def create_initial_files_xy(output_files):
+ for o in output_files:
+ with open(o, "w") as oo: pass
+ # @product
+ @product( create_initial_files_ab, # Input
+ formatter("(.start)$"), # match input file set # 1
- #
- # Run
- #
- pipeline_run(verbose=0)
+ create_initial_files_pq, # Input
+ formatter("(.start)$"), # match input file set # 2
+
+ create_initial_files_xy, # Input
+ formatter("(.start)$"), # match input file set # 3
+
+ "{path[0][0]}/" # Output Replacement string
+ "{basename[0][0]}_vs_" #
+ "{basename[1][0]}_vs_" #
+ "{basename[2][0]}.product", #
+
+ "{path[0][0]}", # Extra parameter: path for 1st set of files, 1st file name
+
+ ["{basename[0][0]}", # Extra parameter: basename for 1st set of files, 1st file name
+ "{basename[1][0]}", # 2nd
+ "{basename[2][0]}", # 3rd
+ ])
+ def product_task(input_file, output_parameter, shared_path, basenames):
+ print "# basenames = ", " ".join(basenames)
+ print "input_parameter = ", input_file
+ print "output_parameter = ", output_parameter, "\n"
+
+
+ #
+ # Run
+ #
+ #pipeline_printout(verbose=6)
+ pipeline_run(verbose=0)
This results in:
- .. code-block:: pycon
- :emphasize-lines: 2,6,10,14,18,22,26,30
+ .. code-block:: pycon
+ :emphasize-lines: 2,6,10,14,18,22,26,30
- >>> pipeline_run(verbose=0)
+ >>> pipeline_run(verbose=0)
- # basenames = a p x
- input_parameter = ('a.start', 'p.start', 'x.start')
- output_parameter = /home/lg/temp/a_vs_p_vs_x.product
+ # basenames = a p x
+ input_parameter = ('a.start', 'p.start', 'x.start')
+ output_parameter = /home/lg/temp/a_vs_p_vs_x.product
- # basenames = a p y
- input_parameter = ('a.start', 'p.start', 'y.start')
- output_parameter = /home/lg/temp/a_vs_p_vs_y.product
+ # basenames = a p y
+ input_parameter = ('a.start', 'p.start', 'y.start')
+ output_parameter = /home/lg/temp/a_vs_p_vs_y.product
- # basenames = a q x
- input_parameter = ('a.start', 'q.start', 'x.start')
- output_parameter = /home/lg/temp/a_vs_q_vs_x.product
+ # basenames = a q x
+ input_parameter = ('a.start', 'q.start', 'x.start')
+ output_parameter = /home/lg/temp/a_vs_q_vs_x.product
- # basenames = a q y
- input_parameter = ('a.start', 'q.start', 'y.start')
- output_parameter = /home/lg/temp/a_vs_q_vs_y.product
+ # basenames = a q y
+ input_parameter = ('a.start', 'q.start', 'y.start')
+ output_parameter = /home/lg/temp/a_vs_q_vs_y.product
- # basenames = b p x
- input_parameter = ('b.start', 'p.start', 'x.start')
- output_parameter = /home/lg/temp/b_vs_p_vs_x.product
+ # basenames = b p x
+ input_parameter = ('b.start', 'p.start', 'x.start')
+ output_parameter = /home/lg/temp/b_vs_p_vs_x.product
- # basenames = b p y
- input_parameter = ('b.start', 'p.start', 'y.start')
- output_parameter = /home/lg/temp/b_vs_p_vs_y.product
+ # basenames = b p y
+ input_parameter = ('b.start', 'p.start', 'y.start')
+ output_parameter = /home/lg/temp/b_vs_p_vs_y.product
- # basenames = b q x
- input_parameter = ('b.start', 'q.start', 'x.start')
- output_parameter = /home/lg/temp/b_vs_q_vs_x.product
+ # basenames = b q x
+ input_parameter = ('b.start', 'q.start', 'x.start')
+ output_parameter = /home/lg/temp/b_vs_q_vs_x.product
- # basenames = b q y
- input_parameter = ('b.start', 'q.start', 'y.start')
- output_parameter = /home/lg/temp/b_vs_q_vs_y.product
+ # basenames = b q y
+ input_parameter = ('b.start', 'q.start', 'y.start')
+ output_parameter = /home/lg/temp/b_vs_q_vs_y.product
**Parameters:**
-.. _decorators.product.tasks_or_file_names:
+.. _decorators.product.input:
- * *tasks_or_file_names*
+ * **input** = *tasks_or_file_names*
can be a:
- #. Task / list of tasks (as in the example above).
- File names are taken from the output of the specified task(s)
+ #. Task / list of tasks.
+ File names are taken from the |output|_ of the specified task(s)
#. (Nested) list of file name strings.
File names containing ``*[]?`` will be expanded as a |glob|_.
E.g.:``"a.*" => "a.1", "a.2"``
+.. _decorators.product.filter:
+
.. _decorators.product.matching_formatter:
- * *matching_formatter*
+ * **filter** = *formater(...)*
a :ref:`formatter<decorators.formatter>` indicator object containing optionally
a python `regular expression (re) <http://docs.python.org/library/re.html>`_.
-.. _decorators.product.output_pattern:
+.. _decorators.product.input2:
+
+.. _decorators.product.filter2:
+
+Additional **input** and **filter** as needed:
+
+ * **input2** = *tasks_or_file_names*
+
+ * **filter2** = *formater(...)*
+
+
+.. _decorators.product.output:
+
+ * **output** = *output*
+ Specifies the resulting output file name(s) after string substitution
+
- * *output_pattern*
- Specifies the resulting output file name(s) after string
- substitution
+.. _decorators.product.extras:
+ * **extras** = *extras*
+ Any extra parameters are passed verbatim to the task function
-.. _decorators.product.extra_parameters:
+ If you are using named parameters, these can be passed as a list, i.e. ``extras= [...]``
- * *extra_parameters*
- Optional extra parameters are passed to the functions after string
- substitution
+ Any extra parameters are consumed by the task function and not forwarded further down the pipeline.
diff --git a/doc/decorators/split.rst b/doc/decorators/split.rst
index e0a6f5f..8151d47 100644
--- a/doc/decorators/split.rst
+++ b/doc/decorators/split.rst
@@ -3,29 +3,35 @@
.. index::
pair: @split; Syntax
+.. role:: raw-html(raw)
+ :format: html
+
+:raw-html:`<style> .red {color:red} </style>`
+
+.. role:: red
+
+
+
.. seealso::
+ * :ref:`@split <new_manual.split>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
+########################################################################
+ at split ( |input|_, |output|_, [|extras|_,...] )
+########################################################################
-########################
- at split
-########################
-
-.. |tasks_or_file_names| replace:: `tasks_or_file_names`
-.. _tasks_or_file_names: `decorators.split.tasks_or_file_names`_
-.. |extra_parameters| replace:: `extra_parameters`
-.. _extra_parameters: `decorators.split.extra_parameters`_
-.. |output_files| replace:: `output_files`
-.. _output_files: `decorators.split.output_files`_
+.. |input| replace:: `input`
+.. _input: `decorators.split.input`_
+.. |extras| replace:: `extras`
+.. _extras: `decorators.split.extras`_
+.. |output| replace:: `output`
+.. _output: `decorators.split.output`_
-*****************************************************************************************************************************************
-*@split* ( |tasks_or_file_names|_, |output_files|_, [|extra_parameters|_,...] )
-*****************************************************************************************************************************************
**Purpose:**
- | Splits a single set of input files into multiple output file names, where the number of
- output files may not be known beforehand.
- | Only out of date tasks (comparing input and output files) will be run
+ | Splits a single set of |input|_ into multiple |output|_, where the number of
+ |output|_ may not be known beforehand.
+ | Only out of date tasks (comparing |input|_ and |output|_ files) will be run
**Example**::
@@ -44,10 +50,10 @@
**Parameters:**
-.. _decorators.split.tasks_or_file_names:
+.. _decorators.split.input:
- * *tasks_or_file_names*
+ * **input** = *tasks_or_file_names*
can be a:
#. (Nested) list of file name strings (as in the example above).
@@ -60,10 +66,12 @@
File names are taken from the output of the specified task(s)
-.. _decorators.split.output_files:
+.. _decorators.split.output:
+
+ * **output** = *output*
+ Specifies the resulting output file name(s) after string substitution
- * *output_files*
- Specifies the resulting output file name(s).
+ Can include glob patterns (e.g. ``"*.txt"``)
| These are used **only** to check if the task is up to date.
| Normally you would use either a |glob|_ (e.g. ``*.little_files`` as above) or a "sentinel file"
@@ -77,16 +85,20 @@
pass
-.. _decorators.split.extra_parameters:
+.. _decorators.split.extras:
- * [*extra_parameters, ...*]
- Any extra parameters are passed verbatim to the task function
+ * **extras** = *extras*
+ Any extra parameters are passed verbatim to the task function
+ If you are using named parameters, these can be passed as a list, i.e. ``extras= [...]``
+ Any extra parameters are consumed by the task function and not forwarded further down the pipeline.
-########################################################################
- at split with ``regex(...)``, ``add_inputs`` and ``inputs``
-########################################################################
- This deprecated syntax is a synonym for :ref:`@subdivide <decorators.subdivide>`.
+
+.. warning::
+
+ Deprecated since Ruffus v 2.5
+
+ :red:`@split( input, output, filter =` ``regex(...)``, ``add_inputs(...)`` | ``inputs(...)``, :red:`[|extras|_,...] )` is a synonym for :ref:`@subdivide <decorators.subdivide>`.
diff --git a/doc/decorators/subdivide.rst b/doc/decorators/subdivide.rst
index b8df277..99ad024 100644
--- a/doc/decorators/subdivide.rst
+++ b/doc/decorators/subdivide.rst
@@ -5,18 +5,19 @@
.. seealso::
+ * :ref:`@subdivide <new_manual.subdivide>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
########################
@subdivide
########################
-.. |tasks_or_file_names| replace:: `tasks_or_file_names`
-.. _tasks_or_file_names: `decorators.subdivide.tasks_or_file_names`_
-.. |extra_parameters| replace:: `extra_parameters`
-.. _extra_parameters: `decorators.subdivide.extra_parameters`_
-.. |output_pattern| replace:: `output_pattern`
-.. _output_pattern: `decorators.subdivide.output_pattern`_
+.. |input| replace:: `input`
+.. _input: `decorators.subdivide.input`_
+.. |extras| replace:: `extras`
+.. _extras: `decorators.subdivide.extras`_
+.. |output| replace:: `output`
+.. _output: `decorators.subdivide.output`_
.. |matching_regex| replace:: `matching_regex`
.. _matching_regex: `decorators.subdivide.matching_regex`_
.. |matching_formatter| replace:: `matching_formatter`
@@ -26,7 +27,7 @@
************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
-*@subdivide* ( |tasks_or_file_names|_, :ref:`regex<decorators.regex>`\ *(*\ |matching_regex|_\ *)* | :ref:`formatter<decorators.formatter>`\ *(*\ |matching_formatter|_\ *)*\, [ :ref:`inputs<decorators.inputs>` *(*\ |input_pattern_or_glob|_\ *)* | :ref:`add_inputs<decorators.add_inputs>` *(*\ |input_pattern_or_glob|_\ *)* ], |output_pattern|_, [|extra_parameters|_,...] )
+*@subdivide* ( |input|_, :ref:`regex<decorators.regex>`\ *(*\ |matching_regex|_\ *)* | :ref:`formatter<decorators.formatter>`\ *(*\ |matching_formatter|_\ *)*\, [ :ref:`inputs<decorators.inputs>` *(*\ |input_pattern_or_glob|_\ *)* | :ref:`add_inputs<decorators.add_inputs>` *(*\ |input_pattern_or_glob|_\ *)* ], |output|_, [|extras|_,...] )
************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
**Purpose:**
@@ -36,7 +37,7 @@
* The number of files in each *Output* can be set at runtime by the use of globs
- * Output file names are specified using the :ref:`formatter<decorators.formatter>` or :ref:`regex<decorators.regex>` indicators from |tasks_or_file_names|_, i.e. from the output
+ * Output file names are specified using the :ref:`formatter<decorators.formatter>` or :ref:`regex<decorators.regex>` indicators from |input|_, i.e. from the output
of specified tasks, or a list of file names, or a |glob|_ matching pattern.
* Additional inputs or dependencies can be added dynamically to the task:
@@ -133,7 +134,7 @@
**Parameters:**
-.. _decorators.subdivide.tasks_or_file_names:
+.. _decorators.subdivide.input:
* *tasks_or_file_names*
can be a:
@@ -159,12 +160,13 @@
a :ref:`formatter<decorators.formatter>` indicator object containing optionally
a python `regular expression (re) <http://docs.python.org/library/re.html>`_.
-.. _decorators.subdivide.output_pattern:
+.. _decorators.subdivide.output:
+
+ * **output** = *output*
+ Specifies the resulting output file name(s) after string substitution
+
+ Can include glob patterns.
- * *output_pattern*
- Specifies the resulting output file name(s). Can include glob patterns.
- Strings are subject to :ref:`regex<decorators.regex>` or :ref:`formatter<decorators.formatter>`
- substitution.
.. _decorators.subdivide.input_pattern_or_glob:
@@ -181,9 +183,14 @@
Strings are subject to :ref:`regex<decorators.regex>` or :ref:`formatter<decorators.formatter>` substitution.
-.. _decorators.subdivide.extra_parameters:
+.. _decorators.subdivide.extras:
+
+ * **extras** = *extras*
+ Any extra parameters are passed verbatim to the task function
+
+ If you are using named parameters, these can be passed as a list, i.e. ``extras= [...]``
+
+ Any extra parameters are consumed by the task function and not forwarded further down the pipeline.
- * *extra_parameters*
- Any extra parameters are consumed by the task function and not forwarded further down the pipeline.
- Strings are subject to :ref:`regex<decorators.regex>` or :ref:`formatter<decorators.formatter>`
- substitution.
+ Strings are subject to :ref:`regex<decorators.regex>` or :ref:`formatter<decorators.formatter>`
+ substitution.
diff --git a/doc/decorators/transform.rst b/doc/decorators/transform.rst
index 494674e..40af901 100644
--- a/doc/decorators/transform.rst
+++ b/doc/decorators/transform.rst
@@ -5,18 +5,18 @@
.. seealso::
+ * :ref:`@transform <new_manual.transform>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
-########################
- at transform
-########################
-.. |tasks_or_file_names| replace:: `tasks_or_file_names`
-.. _tasks_or_file_names: `decorators.transform.tasks_or_file_names`_
-.. |extra_parameters| replace:: `extra_parameters`
-.. _extra_parameters: `decorators.transform.extra_parameters`_
-.. |output_pattern| replace:: `output_pattern`
-.. _output_pattern: `decorators.transform.output_pattern`_
+.. |input| replace:: `input`
+.. _input: `decorators.transform.input`_
+.. |extras| replace:: `extras`
+.. _extras: `decorators.transform.extras`_
+.. |output| replace:: `output`
+.. _output: `decorators.transform.output`_
+.. |filter| replace:: `filter`
+.. _filter: `decorators.transform.filter`_
.. |matching_regex| replace:: `matching_regex`
.. _matching_regex: `decorators.transform.matching_regex`_
.. |matching_formatter| replace:: `matching_formatter`
@@ -24,25 +24,26 @@
.. |suffix_string| replace:: `suffix_string`
.. _suffix_string: `decorators.transform.suffix_string`_
-******************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
-*@transform* ( |tasks_or_file_names|_, :ref:`suffix<decorators.suffix>`\ *(*\ |suffix_string|_\ *)*\ | :ref:`regex<decorators.regex>`\ *(*\ |matching_regex|_\ *)* | :ref:`formatter<decorators.formatter>`\ *(*\ |matching_formatter|_\ *)*\, |output_pattern|_, [|extra_parameters|_,...] )
-******************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
+################################################################################################
+ at transform( |input|_, |filter|_, |output|_, [|extras|_,...] )
+################################################################################################
+
**Purpose:**
- Applies the task function to transform data from input to output files.
+ Applies the task function to transform data from |input|_ to |output|_ files.
- Output file names are specified from |tasks_or_file_names|_, i.e. from the output
+ Output file names are specified from |input|_, i.e. from the |output|_
of specified tasks, or a list of file names, or a |glob|_ matching pattern.
- String replacement occurs either through suffix matches via :ref:`suffix<decorators.suffix>` or
+ String replacement occurs either through suffix matches via :ref:`suffix<decorators.suffix>` or
the :ref:`formatter<decorators.formatter>` or :ref:`regex<decorators.regex>` indicators.
- Only out of date tasks (comparing input and output files) will be run
+ Only out of date tasks (comparing |input|_ and |output|_ files) will be run
**Simple Example**
Transforms ``*.c`` to ``*.o``::
- @transform(["1.c", "2.c"], suffix(".c"), ".o")
+ @transform(input = ["1.c", "2.c"], filter = suffix(".c"), output = ".o")
def compile(infile, outfile):
pass
@@ -71,22 +72,25 @@
**Parameters:**
-.. _decorators.transform.tasks_or_file_names:
+.. _decorators.transform.input:
- * *tasks_or_file_names*
+ * **input** = *tasks_or_file_names*
can be a:
#. Task / list of tasks (as in the example above).
- File names are taken from the output of the specified task(s)
+ File names are taken from the |output|_ of the specified task(s)
#. (Nested) list of file name strings.
File names containing ``*[]?`` will be expanded as a |glob|_.
E.g.:``"a.*" => "a.1", "a.2"``
+
+.. _decorators.transform.filter:
+
.. _decorators.transform.suffix_string:
- * *suffix_string*
+ * **filter** = *suffix(suffix_string)*
must be wrapped in a :ref:`suffix<decorators.suffix>` indicator object.
- The end of each input file name which matches ``suffix_string`` will be replaced by ``output_pattern``.
+ The end of each |input|_ file name which matches ``suffix_string`` will be replaced by |output|_.
Input file names which do not match suffix_string will be ignored
@@ -132,28 +136,32 @@
.. _decorators.transform.matching_regex:
- * *matching_regex*
+ * **filter** = *regex(matching_regex)*
is a python regular expression string, which must be wrapped in
a :ref:`regex<decorators.regex>`\ indicator object
See python `regular expression (re) <http://docs.python.org/library/re.html>`_
documentation for details of regular expression syntax
- Each output file name is created using regular expression substitution with ``output_pattern``
+ Each output file name is created using regular expression substitution with ``output``
.. _decorators.transform.matching_formatter:
- * *matching_formatter*
+ * **filter** = *formatter(...)*
a :ref:`formatter<decorators.formatter>` indicator object containing optionally
- a python `regular expression (re) <http://docs.python.org/library/re.html>`_.
+ a python `regular expression (re) <http://docs.python.org/library/re.html>`_.
+
+.. _decorators.transform.output:
+
+ * **output** = *output*
+ Specifies the resulting |output|_ file name(s) after string substitution
-.. _decorators.transform.output_pattern:
+.. _decorators.transform.extras:
- * *output_pattern*
- Specifies the resulting output file name(s).
+ * **extras** = *extras*
+ Any extra parameters are passed verbatim to the task function
-.. _decorators.transform.extra_parameters:
+ If you are using named parameters, these can be passed as a list, i.e. ``extras= [...]``
- * [*extra_parameters, ...*]
- Any extra parameters are passed to the task function.
+ Any extra parameters are consumed by the task function and not forwarded further down the pipeline.
If ``regex(matching_regex)`` or ``formatter(...)``` is used, then substitution
is first applied to (even nested) string parameters. Other data types are passed
diff --git a/doc/decorators/transform_ex.rst b/doc/decorators/transform_ex.rst
index 2436696..c0b0054 100644
--- a/doc/decorators/transform_ex.rst
+++ b/doc/decorators/transform_ex.rst
@@ -7,59 +7,56 @@
.. seealso::
+ * :ref:`@transform(.., add_inputs(...)| inputs(...), ...) <new_manual.inputs>` in the **Ruffus** Manual
* :ref:`Decorators <decorators>` for more decorators
-####################################################
- at transform with ``add_inputs`` and ``inputs``
-####################################################
-
-.. |tasks_or_file_names| replace:: `tasks_or_file_names`
-.. _tasks_or_file_names: `decorators.transform.tasks_or_file_names`_
-.. |extra_parameters| replace:: `extra_parameters`
-.. _extra_parameters: `decorators.transform.extra_parameters`_
-.. |output_pattern| replace:: `output_pattern`
-.. _output_pattern: `decorators.transform.output_pattern`_
-.. |input_pattern_or_glob| replace:: `input_pattern_or_glob`
-.. _input_pattern_or_glob: `decorators.transform.input_pattern_or_glob`_
+.. |input| replace:: `input`
+.. _input: `decorators.transform_ex.input`_
+.. |filter| replace:: `filter`
+.. _filter: `decorators.transform_ex.filter`_
+.. |extras| replace:: `extras`
+.. _extras: `decorators.transform_ex.extras`_
+.. |output| replace:: `output`
+.. _output: `decorators.transform_ex.output`_
.. |matching_regex| replace:: `matching_regex`
-.. _matching_regex: `decorators.transform.matching_regex`_
+.. _matching_regex: `decorators.transform_ex.matching_regex`_
.. |matching_formatter| replace:: `matching_formatter`
-.. _matching_formatter: `decorators.transform.matching_formatter`_
+.. _matching_formatter: `decorators.transform_ex.matching_formatter`_
.. |suffix_string| replace:: `suffix_string`
-.. _suffix_string: `decorators.transform.suffix_string`_
+.. _suffix_string: `decorators.transform_ex.suffix_string`_
+.. |replace_inputs| replace:: `replace_inputs`
+.. _replace_inputs: `decorators.transform_ex.replace_inputs`_
+.. |add_inputs| replace:: `add_inputs`
+.. _add_inputs: `decorators.transform_ex.add_inputs`_
+################################################################################################################################################
+ at transform( |input|_, |filter|_, |replace_inputs|_ | |add_inputs|_, |output|_, [|extras|_,...] )
+################################################################################################################################################
-
-
-************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************ [...]
-*@transform* ( |tasks_or_file_names|_, :ref:`suffix<decorators.suffix>`\ *(*\ |suffix_string|_\ *)*\ | :ref:`regex<decorators.regex>`\ *(*\ |matching_regex|_\ *)* | :ref:`formatter<decorators.formatter>`\ *(*\ |matching_formatter|_\ *)*\, :ref:`inputs<decorators.inputs>` | :ref:`add_inputs<decorators.add_inputs>`\ *(*\ |input_pattern_or_glob|_\ *)*\ , |output_pattern|_, [|extra_parameters|_,...] )
-************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************ [...]
**Purpose:**
+ Applies the task function to transform data from |input|_ to |output|_ files.
+
This variant of ``@transform`` allows additional inputs or dependencies to be added
dynamically to the task.
- Output file names and strings in the extra parameters
- are determined from |tasks_or_file_names|_, i.e. from the output
- of up stream tasks, or a list of file names.
+ Output file names are specified from |input|_, i.e. from the |output|_
+ of specified tasks, or a list of file names, or a |glob|_ matching pattern.
- This variant of ``@transform`` allows input file names to be derived in the same way.
+ This variant of ``@transform`` allows additional or replacement input file names to be derived in the same way.
- String replacement occurs either through suffix matches via :ref:`suffix<decorators.suffix>` or
+ String replacement occurs either through suffix matches via :ref:`suffix<decorators.suffix>` or
the :ref:`formatter<decorators.formatter>` or :ref:`regex<decorators.regex>` indicators.
- ``@collate`` groups together all **Input** which result in identical **Output** and **extra**
- parameters.
+ It is a **one to one** operation.
- It is a **many to fewer** operation.
+ :ref:`add_inputs(...)<decorators.add_inputs>` nests the the original input parameters in a list before adding additional dependencies.
- :ref:`add_inputs<decorators.add_inputs>` nests the the original input parameters in a list before adding additional dependencies.
-
- :ref:`inputs<decorators.inputs>` replaces the original input parameters wholescale.
+ :ref:`inputs(...)<decorators.inputs>` replaces the original input parameters wholescale.
Only out of date tasks (comparing input and output files) will be run
- **Example of** :ref:`add_inputs<decorators.add_inputs>`
+ **Example of** :ref:`add_inputs(...)<decorators.add_inputs>`
A common task in compiling C code is to include the corresponding header file for the source.
@@ -76,16 +73,18 @@
compile(["1.c", "1.h", "universal.h"], "1.o")
compile(["2.c", "2.h", "universal.h"], "2.o")
- **Example of** :ref:`inputs<decorators.inputs>`
+ **Example of** :ref:`inputs(...)<decorators.inputs>`
- ``inputs(...)`` allows the original input parameters to be replaced wholescale.
+ :ref:`inputs(...)<decorators.inputs>` allows the original input parameters to be replaced wholescale.
This can be seen in the following example:
::
- @transform([ ["1.c", "A.c", 2]
- ["2.c", "B.c", "C.c", 3]],
- suffix(".c"), inputs([r"\1.py", "docs.rst"]), ".pyc")
+ @transform(input = [ ["1.c", "A.c", 2]
+ ["2.c", "B.c", "C.c", 3]],
+ filter = suffix(".c"),
+ replace_inputs = inputs([r"\1.py", "docs.rst"]),
+ output = ".pyc")
def compile(infile, outfile):
pass
@@ -99,25 +98,29 @@
**Parameters:**
-.. _decorators.transform.tasks_or_file_names:
+.. _decorators.transform_ex.input:
- * *tasks_or_file_names*
+ * **input** = *tasks_or_file_names*
can be a:
#. Task / list of tasks (as in the example above).
- File names are taken from the output of the specified task(s)
+ File names are taken from the |output|_ of the specified task(s)
#. (Nested) list of file name strings.
File names containing ``*[]?`` will be expanded as a |glob|_.
E.g.:``"a.*" => "a.1", "a.2"``
-.. _decorators.transform.suffix_string:
+.. _decorators.transform_ex.filter:
+
+.. _decorators.transform_ex.suffix_string:
- * *suffix_string*
+ * **filter** = *suffix(suffix_string)*
must be wrapped in a :ref:`suffix<decorators.suffix>` indicator object.
- The end of each file name which matches suffix_string will be replaced by `output_pattern`.
+ The end of each |input|_ file name which matches ``suffix_string`` will be replaced by |output|_.
Thus::
- @transform(["a.c", "b.c"], suffix(".c"), ".o")
+ @transform(input = ["a.c", "b.c"],
+ filter = suffix(".c"),
+ output = ".o")
def compile(infile, outfile):
pass
@@ -128,47 +131,56 @@
File names which do not match suffix_string will be ignored
-.. _decorators.transform.matching_regex:
+.. _decorators.transform_ex.matching_regex:
- * *matching_regex*
+ * **filter** = *regex(matching_regex)*
is a python regular expression string, which must be wrapped in
a :ref:`regex<decorators.regex>` indicator object
See python `regular expression (re) <http://docs.python.org/library/re.html>`_
documentation for details of regular expression syntax
- Each output file name is created using regular expression substitution with ``output_pattern``
+ Each output file name is created using regular expression substitution with ``output``
-.. _decorators.transform.matching_formatter:
+.. _decorators.transform_ex.matching_formatter:
- * *matching_formatter*
+ * **filter** = *formatter(...)*
a :ref:`formatter<decorators.formatter>` indicator object containing optionally
a python `regular expression (re) <http://docs.python.org/library/re.html>`_.
-.. _decorators.transform.input_pattern_or_glob:
- * *input_pattern*
- Specifies the resulting input(s) to each job.
- Must be wrapped in an :ref:`inputs<decorators.inputs>` or an :ref:`inputs<decorators.add_inputs>` indicator object.
+.. _decorators.transform_ex.add_inputs:
- Can be a:
+.. _decorators.transform_ex.replace_inputs:
- #. Task / list of tasks (as in the example above).
+ * **add_inputs** = *add_inputs*\ (...) or **replace_inputs** = *inputs*\ (...)
+ Specifies the resulting |input|_\ (s) to each job.
+
+ Positional parameters must be disambiguated by wrapping the values in :ref:`inputs(...)<decorators.inputs>` or an :ref:`add_inputs(...)<decorators.add_inputs>`.
+
+ Named parameters can be passed the values directly.
+
+ Takes:
+
+ #. Task / list of tasks.
File names are taken from the output of the specified task(s)
#. (Nested) list of file name strings.
Strings will be subject to substitution.
File names containing ``*[]?`` will be expanded as a |glob|_.
- E.g.:``"a.*" => "a.1", "a.2"``
+ E.g. ``"a.*" => "a.1", "a.2"``
+
+.. _decorators.transform_ex.output:
+ * **output** = *output*
+ Specifies the resulting |output|_ file name(s) after string substitution
-.. _decorators.transform.output_pattern:
+.. _decorators.transform_ex.extras:
- * *output_pattern*
- Specifies the resulting output file name(s).
+ * **extras** = *extras*
+ Any extra parameters are passed verbatim to the task function
-.. _decorators.transform.extra_parameters:
+ If you are using named parameters, these can be passed as a list, i.e. ``extras= [...]``
- * [*extra_parameters, ...*]
- Any extra parameters are passed to the task function.
+ Any extra parameters are consumed by the task function and not forwarded further down the pipeline.
If the ``regex(...)`` or ``formatter(...)`` parameter is used, then substitution
is first applied to (even nested) string parameters. Other data types are passed
diff --git a/doc/examples/bioinformatics/part2.rst b/doc/examples/bioinformatics/part2.rst
index 30e1343..554e588 100644
--- a/doc/examples/bioinformatics/part2.rst
+++ b/doc/examples/bioinformatics/part2.rst
@@ -32,7 +32,7 @@ Step 1. Cleaning up any leftover junk from previous pipeline runs
hanging around as an unwanted, extraneous and confusing orphan.
As a general rule, it is a good idea to clean up the results of a previous run in
- a :ref:`@split <manual.split>` operation:
+ a :ref:`@split <new_manual.split>` operation:
::
diff --git a/doc/faq.rst b/doc/faq.rst
index 2b70f66..8886b3a 100644
--- a/doc/faq.rst
+++ b/doc/faq.rst
@@ -210,18 +210,18 @@ Q. How can I use my own decorators with Ruffus?
A. With care! If the following two points are observed:
-____________________________________________________________________________________________________________________________________________________________________________________________________________________
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
1. Use `@wraps <https://docs.python.org/2/library/functools.html#functools.wraps>`__ from ``functools`` or Michele Simionato's `decorator <https://pypi.python.org/pypi/decorator>`__ module
-____________________________________________________________________________________________________________________________________________________________________________________________________________________
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
These will automatically forward attributes from the task function correctly:
* ``__name__`` and ``__module__`` is used to identify functions uniquely in a Ruffus pipeline, and
* ``pipeline_task`` is used to hold per task data
-__________________________________________________________________________________________________________
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
2. Always call Ruffus decorators first before your own decorators.
-__________________________________________________________________________________________________________
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Otherwise, your decorator will be ignored.
@@ -257,9 +257,9 @@ ________________________________________________________________________________
-_____________________________________________________
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Example decorator:
-_____________________________________________________
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Let us look at a decorator to time jobs:
@@ -304,9 +304,9 @@ _____________________________________________________
What would ``@time_job`` look like?
-__________________________________________________________________________________________________________
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
1. Using functools `@wraps <https://docs.python.org/2/library/functools.html#functools.wraps>`__
-__________________________________________________________________________________________________________
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
.. code-block:: python
@@ -320,9 +320,9 @@ ________________________________________________________________________________
return wrapper
return actual_time_job
-__________________________________________________________________________________________________________
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
2. Using Michele Simionato's `decorator <https://pypi.python.org/pypi/decorator>`__ module
-__________________________________________________________________________________________________________
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
.. code-block:: python
@@ -334,9 +334,9 @@ ________________________________________________________________________________
return decorator.decorator(time_job)
-_______________________________________________________________________________________________________________________________________________________________
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
2. By hand, using a `callable object <https://docs.python.org/2/reference/datamodel.html#emulating-callable-objects>`__
-_______________________________________________________________________________________________________________________________________________________________
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
.. code-block:: python
@@ -540,9 +540,9 @@ Q. Can I call extra code before each job?
Q. Does *Ruffus* allow checkpointing: to distinguish interrupted and completed results?
=========================================================================================================
-_____________________________________________________
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
A. Use the builtin sqlite checkpointing
-_____________________________________________________
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
By default, ``pipeline_run(...)`` will save the timestamps for output files from successfully run jobs to an sqlite database file (``.ruffus_history.sqlite``) in the current directory .
@@ -564,9 +564,9 @@ _____________________________________________________
level 2 : above, plus a checksum of the pipeline function body
level 3 : above, plus a checksum of the pipeline function default arguments and the additional arguments passed in by task decorators
-_____________________________________________________
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
A. Use a flag file
-_____________________________________________________
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
When gmake is interrupted, it will delete the target file it is updating so that the target is
remade from scratch when make is next run. Ruffus, by design, does not do this because, more often than
@@ -636,9 +636,9 @@ _____________________________________________________
The :ref:`Bioinformatics example<examples_bioinformatics_part2.step2>` contains :ref:`code <examples_bioinformatics_part2_code>` for checkpointing.
-_____________________________________________________
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
A. Use a temp file
-_____________________________________________________
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Thanks to Martin Goodson for suggesting this and providing an example. In his words:
diff --git a/doc/global.inc b/doc/global.inc
index 034a3b6..dd60e3e 100644
--- a/doc/global.inc
+++ b/doc/global.inc
@@ -1,4 +1,13 @@
.. |glob| replace:: `glob`
.. _glob: http://docs.python.org/library/glob.html
-
+.. meta::
+ :description: Ruffus is a Computation Pipeline library for python. It is open-sourced, powerful and user-friendly, and widely used in science and bioinformatics
+ :keywords: python, pipeline, computation, cluster, Ruffus, bioinformatics, parallel' name='keywords
+ :Author: Leo Goodstadt
+ :Publisher: Leo Goodstadt
+ :Email: ruffus at llew.org.uk
+ :robots: index, follow
+ :language: en-uk
+ :country: United Kingdom
+ :copyright: Leo Goodstadt 2009-2014
diff --git a/doc/history.rst b/doc/history.rst
index c0a8f05..2eb2031 100644
--- a/doc/history.rst
+++ b/doc/history.rst
@@ -1,5 +1,12 @@
.. include:: global.inc
+.. role:: raw-html(raw)
+ :format: html
+
+:raw-html:`<style> .red {color:red} </style>`
+
+.. role:: red
+
########################################
Major Features added to Ruffus
@@ -9,14 +16,173 @@ Major Features added to Ruffus
See :ref:`To do list <todo>` for future enhancements to Ruffus
-
********************************************************************
-version 2.5RC
+version 2.6
********************************************************************
- 31st July 2014: Release Candidate
+ 12th March 2015
+
+=====================================================================================================================
+1) Bug fixes
+=====================================================================================================================
+
+ * ``pipeline_printout_graph()`` incompatibility with python3 fixed
+ * checkpointing did not work correctly with :ref:`@split(...) <decorators.split>` and :ref:`@subdivide(...) <decorators.subdivide>`
+
+
+=====================================================================================================================
+2) `@transform `(..., suffix("xxx"),` :red:`output_dir` `= "/new/output/path")`
+=====================================================================================================================
+
+ Thanks to the suggestion of Milan Simonovic.
+
+ :ref:`@transform(..., suffix(...) ) <decorators.transform>` has easy to understand syntax and takes care of all the common use cases
+ of Ruffus.
+
+ However, when we need to place the output in a different directories, we suddenly have to plunge into the deep end and parse file paths using
+ :ref:`regex() <decorators.regex>` or :ref:`formatter() <new_manual.formatter>`.
+
+ Now, :ref:`@transform <decorators.transform>` takes an optional ``output_dir`` named parameter so that we can continue to use :ref:`suffix() <new_manual.suffix>` even when the output needs
+ to go into a new directory.
+
+ .. <<Python
+
+ .. code-block:: python
+ :emphasize-lines: 2,3,9
+
+ #
+ # input/a.fasta -> output/a.sam
+ # input/b.fasta -> output/b.sam
+ #
+ starting_files = ["input/a.fasta","input/b.fasta"]
+ @transform(starting_files,
+ suffix('.fasta'),
+ '.sam',
+ output_dir = "output")
+ def map_dna_sequence(input_file, output_file) :
+ pass
+
+ ..
+ Python
+
+ See example ``test\test_suffix_output_dir.py``
+
+=====================================================================================================================
+2) Named parameters
+=====================================================================================================================
+
+ Decorators can take named parameters.
+
+ These are self documenting, and improve clarity.
+
+ Note that the usual Python rules for function parameters apply:
+
+ * Positional arguments must precede named arguments
+ * Named arguments cannot be used to fill in for "missing" positional arguments
+
+
+ For example the following two functions are identical:
+
+ **Positional parameters:**
+
+ .. <<Python
+
+ .. code-block:: python
+
+ @merge(prev_task, ["a.summary", "b.summary"], 14, "extra_info", {"a":45, "b":5})
+ def merge_task(inputs, outputs, extra_num, extra_str, extra_dict):
+ pass
+ ..
+ Python
+
+ **Named parameters:**
+
+ .. <<Python
+
+ .. code-block:: python
+
+ # new style is a bit clearer
+ @merge(input = prev_task,
+ output = ["a.summary", "b.summary"],
+ extras = [14, "extra_info", {"a":45, "b":5}]
+ )
+ def merge_task(inputs, outputs, extra_num, extra_str, extra_dict):
+ pass
+ ..
+ Python
+
+ .. warning::
+
+ ``,extras=`` takes all the *extras* parameters (``14, "extra_info", {"a":45, "b":5}``) as a single list
+
+ * :ref:`@split(...) <decorators.split>` and :ref:`@merge(...) <decorators.merge>`
+ * *input*
+ * *output*
+ * [*extras*\ ]
+ * :ref:`@transform(...) <decorators.transform>` and :ref:`@mkdir(...) <decorators.mkdir>`
+ * *input*
+ * *filter*
+ * [*replace_inputs* or *add_inputs*\ ]
+ * *output*
+ * [*extras*\ ]
+ * [*output_dir*\ ]
+ * :ref:`@collate(...) <decorators.collate>` and :ref:`@subdivide(...) <decorators.collate>`
+ * *input*
+ * *filter*
+ * *output*
+ * [*extras*\ ]
+ * :ref:`@originate(...) <decorators.originate>`
+ * *output*
+ * [*extras*\ ]
+ * :ref:`@product(...) <decorators.product>`, :ref:`@permutations(...) <decorators.permutations>`, :ref:`@combinations(...) <decorators.combinations>`, and :ref:`@combinations_with_replacement(...) <decorators.combinations_with_replacement>`
+ * *input*
+ * *filter*
+ * [*input2...NNN*\ ] (only for ``product``)
+ * [*filter2...NNN*\ ] (only for ``product``) where NNN is an incrementing number
+ * *tuple_size* (except for ``product``)
+ * [*replace_inputs* or *add_inputs*\ ]
+ * *output*
+ * [*extras*\ ]
+
+
+
+=============================================
+3) New object orientated syntax for Ruffus
+=============================================
+
+ Ruffus Pipelines can now be created directly using the new ``Pipeline`` and ``Task`` objects instead of via decorators.
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 9
+
+ # make ruffus pipeline
+ my_pipeline = Pipeline(name = "test")
+ my_pipeline.transform(task_func = map_dna_sequence,
+ input = starting_files,
+ filter = suffix('.fasta'),
+ output = '.sam',
+ output_dir = "output")
+
+ my_pipeline.run()
+ ..
+ python
+
+ This new syntax is fully compatible and inter-operates with traditional Ruffus syntax using decorators.
+
+ Apart from cosmetic changes, the new syntax allows different instances of modular Ruffus sub-pipelines
+ to be defined separately, in different python modules and then joined together flexible at runtime.
+
+ The new syntax and discussion are introduced :ref:`here <new_syntax>`.
+
+
+
+********************************************************************
+version 2.5
+********************************************************************
- 5th August 2014: Release
+ 6th August 2014
============================================================================================================================================================
1) Python3 compatability (but at least python 2.6 is now required)
diff --git a/doc/images/small_logo.png b/doc/images/small_logo.png
new file mode 100644
index 0000000..cd7e4d0
Binary files /dev/null and b/doc/images/small_logo.png differ
diff --git a/doc/images/subpipeline_example.png b/doc/images/subpipeline_example.png
new file mode 100644
index 0000000..5c6fae8
Binary files /dev/null and b/doc/images/subpipeline_example.png differ
diff --git a/doc/implementation_notes.rst b/doc/implementation_notes.rst
index bd08d03..4a87582 100644
--- a/doc/implementation_notes.rst
+++ b/doc/implementation_notes.rst
@@ -3,6 +3,35 @@ Implementation Tips
##########################################
******************************************************************************
+Items remaining for current release
+******************************************************************************
+======================================================================================================
+Code
+======================================================================================================
+ #. update_checksum_level_on_tasks(checksum_level) is non reentrant
+ #. ``Task.description_with_args_placeholder`` needs to only fill in placeholders at the last minute
+ Otherwise cloned pipelines will have the wrong name
+
+======================================================================================================
+Unit tests
+======================================================================================================
+ #. output_dir for @mkdir
+ #. When are things defined / linked up
+ #. When can we join up Pipelines / tasks / set_input()?
+ #. Sub pipeline
+ #. Whether setup occurs ``pipeline_run()`` where ``target_tasks`` and ``forcedtorun_tasks`` are in different linked or unlinked pipelines
+ #. name lookup
+ #. Task (dependency) parsing inside @transform, pipeline.transform(input = , add_inputs, replace_inputs =), pipeline.split(..., output=)
+ #. ``mkdir()`` should not be allowed inside input parameters apart from @follows
+ #. Cannot dependency cannot be self
+ #. ``Pipeline.clone()``
+ #. ``Task.set_input()``
+ #. ``@product`` ``set_input`` should take (``input``, ``input2``...)
+
+
+
+
+******************************************************************************
Release
******************************************************************************
@@ -20,7 +49,7 @@ Release
* tag git with, for example::
- git tag -a v2.5 -m "Version 2.5"
+ git tag -a v2.6 -m "Version 2.6"
* Upload to pypi::
@@ -33,6 +62,23 @@ Release
git push
******************************************************************************
+blogger
+******************************************************************************
+
+ ::
+
+
+ .article-content h2 {color: #ad3a2b}
+ .article-content h3 {color: #0100b4}
+ #header .header-bar .title h1
+ {
+ background-image: url('http://www.ruffus.org.uk/_static/small_logo.png');
+ background-repeat: no-repeat;
+ background-position: left;
+ }
+
+
+******************************************************************************
dbdict.py
******************************************************************************
@@ -416,8 +462,6 @@ Task completion monitoring
Should be fast: a single db connection is created and used inside ``pipeline_run``, ``pipeline_printout``, ``pipeline_printout_graph``
-
-
===================================================
Avoid pauses between tasks
===================================================
@@ -426,17 +470,393 @@ Task completion monitoring
* If the local file time looks to be in sync with the underlying file system, saved system time is used instead of file timestamps
-
-
-
******************************************************************************************
``@mkdir(...),``
******************************************************************************************
- * ``mkdir`` continues to work seamlessly inside ``@follows``) but also as its own decorator ``@mkdir`` due to the original happy orthogonal design
+ * ``mkdir`` continues to work seamlessly inside ``@follows`` but also as its own decorator ``@mkdir`` due to the original happy orthogonal design
* fixed bug in checking so that Ruffus does't blow up if non strings are in the output (number...)
* note: adding the decorator to a previously undecorated function might have unintended consequences. The undecorated function turns into a zombie.
* fixed ugly bug in ``pipeline_printout`` for printing single line output
* fixed description and printout indent
+
+******************************************************************************
+Parameter handling
+******************************************************************************
+
+======================================================================================================
+ Current design
+======================================================================================================
+
+ Parameters in Ruffus v 2.x are obtained using a "pull" model.
+
+ Each task has its self.param_generator_func()
+ This is an iterator function which yields ``param`` and ``descriptive_param`` per iteration:
+
+ .. code-block:: python
+
+ for param, descriptive_param in self.param_generator_func(runtime_data):
+ pass
+
+
+ ``param`` and ``descriptive_param`` are basically the same except that globs are not expanded in ``descriptive_param`` because
+ they are used for display.
+
+
+ The iterator functions have all the state they need to generate their input, output and extra parameters
+ (only ``runtime_data``) is added at run time.
+ These closures are generated as nested functions inside "factory" functions defined in ``file_name_parameters.py``
+
+ Each task type has its own factory function. For example:
+
+ .. code-block:: python
+
+ args_param_factory (orig_args)
+ files_param_factory (input_files_task_globs, flatten_input, do_not_expand_single_job_tasks, output_extras)
+ split_param_factory (input_files_task_globs, output_files_task_globs, *extra_params)
+ merge_param_factory (input_files_task_globs, output_param, *extra_params)
+ originate_param_factory (list_output_files_task_globs, extras)
+
+
+ The following factory files delegate most of their work to ``yield_io_params_per_job``:
+
+ to support:
+
+ * ``inputs()``, ``add_inputs()`` input parameter supplementing
+ * extra inputs, outputs, extra parameter replacement with ``suffix()``, ``regex()`` and ``formatter``
+
+ .. code-block:: python
+
+ collate_param_factory (input_files_task_globs, flatten_input, file_names_transform, extra_input_files_task_globs, replace_inputs, output_pattern, *extra_specs)
+ transform_param_factory (input_files_task_globs, flatten_input, file_names_transform, extra_input_files_task_globs, replace_inputs, output_pattern, *extra_specs)
+ combinatorics_param_factory (input_files_task_globs, flatten_input, combinatorics_type, k_tuple, file_names_transform, extra_input_files_task_globs, replace_inputs, output_pattern, *extra_specs)
+ subdivide_param_factory (input_files_task_globs, flatten_input, file_names_transform, extra_input_files_task_globs, replace_inputs, output_files_task_globs, *extra_specs)
+ product_param_factory (list_input_files_task_globs, flatten_input, file_names_transform, extra_input_files_task_globs, replace_inputs, output_pattern, *extra_specs)
+
+
+ yield_io_params_per_job (input_params, file_names_transform, extra_input_files_task_globs, replace_inputs, output_pattern, extra_specs, runtime_data, iterator, expand_globs_in_output = False):
+
+
+ #. The first thing they do is to get a list of input parameters, either directly, or by expanding globs or by query upstream tasks:
+
+ .. code-block:: python
+
+ file_names_from_tasks_globs(files_task_globs, runtime_data, do_not_expand_single_job_tasks = True_if_split_or_merge)
+
+ .. note ::
+
+ ``True_if_split_or_merge`` is a wierd parameter which directly queries the upstream dependency for its output files if it is a single task...
+
+ This is legacy code. Probably should be refactored out of existence...
+
+
+ #. They then convert the input parameters to a flattened list of file names (passing through unchanged the original input parameters structure)
+
+ .. code-block:: python
+
+ input_param_to_file_name_list()
+ # combinatorics and product call:
+ list_input_param_to_file_name_list()
+
+ This is done at the iterator level because the combinatorics decorators do not have just a
+ list of input parameters (They have combinations, permutations, products of
+ input parameters etc) but a list of lists of input parameters.
+
+ transform, collate, subdivide => list of strings.
+ combinatorics / product => list of lists of strings
+
+ #. ``yield_io_params_per_job`` yields pairs of param sets by
+
+ * Replacing or supplementing input parameters for the indicator objects ``inputs()`` and ``add_inputs()``
+ * Expanding extra parameters
+ * Expanding output parameters (with or without expanding globs)
+
+ In each case:
+ * If these contains objects which look like strings, we do regular expression / file component substitution
+ * If they contain tasks, these are queries for output files
+
+
+ .. note ::
+
+ This should be changed:
+
+ If the flattened list of input file names is empty, ie. if the input parameters
+ contain just other stuff, then the entire parameter is ignored.
+
+======================================================================================================
+ Handling file names
+======================================================================================================
+
+ All strings in input (or output parameters) are treated as file names unless they are wrapped
+ with ``output_from`` in which case they are ``Task``, ``Pipeline`` or function names.
+
+ A list of strings for ready for substitution to output parameters is obtained from the
+ ``ruffus_utility.get_strings_in_flattened_sequence()``
+
+ This is called from:
+
+ file_name_parameters
+
+ (1) Either to check that input files exist:
+ ``check_input_files_exist()``
+ ``needs_update_check_directory_missing()``
+ ``needs_update_check_exist()``
+ ``needs_update_check_modify_time()``
+
+ (2) Or to generate parameters from the various param factories
+
+ ``product_param_factory()``
+ ``transform_param_factory()``
+ ``collate_param_factory()``
+ ``combinatorics_param_factory()``
+ ``subdivide_param_factory()``
+
+ These first call ``file_names_from_tasks_globs()`` to get the input parameters,
+ then pass a flattened list of strings to ``yield_io_params_per_job()``
+
+ -> ``file_names_from_tasks_globs()``
+ -> ``yield_io_params_per_job(`` ``input_param_to_file_name_list()`` / ``list_input_param_to_file_name_list()`` ``)``
+
+
+ task
+
+ (3) to obtain a list of file names to ``touch``
+
+ ``job_wrapper_io_files``
+
+ (4) to make directories
+
+ ``job_wrapper_mkdir``
+
+ (5) update / remove files in ``job_history`` if job succeeded or failed
+
+ ``pipeline_run``
+
+
+======================================================================================================
+ Refactor to handle input parameter objects with ruffus_params() functions
+======================================================================================================
+
+ We want to expand objects with ruffus_params *only* when doing output parameter
+ substitution, i.e. Case (2) above. They are not file names: cases (1), (3), (4), (5).
+
+ Therefore: Expand in ``file_names_from_tasks_globs()`` which also handles
+ ``inputs()`` and ``add_inputs`` and ``@split`` outputs.
+
+======================================================================================================
+ Refactor to handle formatter() replacement with "{EXTRAS[0][1][3]}" and "[INPUTS[1][2]]"
+======================================================================================================
+
+ Non-recursive Substitution in all:
+
+ construct new list where each item is replaced referring to the original and then assign
+
+ extra_inputs() "[INPUTS[1][2]]" refers to the original input
+ output / extras "[INPUTS[1][2]]" refers to substituted input
+
+
+ In addition to the flattened input paramters, we need to pass in the unflattened input and extra parameters
+
+ In ``file_name_parameters.py.``: ``yield_io_params_per_job``
+
+ From:
+ .. code-block:: python
+
+ extra_inputs = extra_input_files_task_globs.file_names_transformed (filenames, file_names_transform)
+ extra_params = tuple( file_names_transform.substitute(filenames, p) for p in extra_specs)
+ output_pattern_transformed = output_pattern.file_names_transformed (filenames, file_names_transform)
+ output_param = file_names_transform.substitute_output_files(filenames, output_pattern)
+
+ To:
+ .. code-block:: python
+
+ extra_inputs = extra_input_files_task_globs.file_names_transformed (orig_input_param, extra_specs, filenames, file_names_transform)
+ extra_params = tuple( file_names_transform.substitute(input_param, extra_specs, filenames, p) for p in extra_specs)
+ output_pattern_transformed = output_pattern.file_names_transformed (input_param, extra_specs, filenames, file_names_transform)
+ output_param = file_names_transform.substitute_output_files(input_param, extra_specs, filenames, output_pattern)
+
+ In other words, we need two extra parameters for inputs and extras
+
+ .. code-block:: python
+
+ class t_file_names_transform(object):
+ def substitute (self, input_param, extra_param, starting_file_names, pattern):
+ pass
+ def substitute_output_files (self, input_param, extra_param, starting_file_names, pattern):
+ pass
+
+
+ class t_params_tasks_globs_run_time_data(object):
+ def file_names_transformed (self, input_param, extra_param, filenames, file_names_transform):
+ pass
+
+
+======================================================================================================
+ Refactor to handle alternative outputs with either_or(...,...)
+======================================================================================================
+
+ * what happens to get_outputs or checkpointing when the job completes but the output files are not made?
+ * either_or matches
+
+ * the only alternative to have all files existing
+ * the alternative with the most recent file
+
+ * either_or behaves as ``list()`` in ``file_name_parameters.py.`` : ``file_names_from_tasks_globs``
+
+
+
+ * Handled to check that input files exist:
+
+ ``check_input_files_exist()``
+ ``needs_update_check_directory_missing()``
+ ``needs_update_check_exist()``
+ ``needs_update_check_modify_time()``
+
+ * Handled to update / remove files in ``job_history`` if job succeeded or failed
+
+ * Only first either_or is used to obtain list of file names to ``touch``
+
+ ``task.job_wrapper_io_files``
+
+ * Only first either_or is used to obtain list of file names to make directories
+
+ ``job_wrapper_mkdir``
+
+ * What happens in ``task.get_output_files()``?
+
+
+******************************************************************************
+ Add Object Orientated interface
+******************************************************************************
+
+
+======================================================================================================
+Passed Unit tests
+======================================================================================================
+ #. Refactored to remove unused "flattened" code paths / parameters
+ #. Prefix all attributes for Task into underscore so that help(Task) is not overloaded with details
+ #. Named parameters
+ * parse named parameters in order filling in from unnamed
+ * save parameters in ``dict`` ``Task.parsed_args``
+ * call ``setup_task_func()`` afterwards which knows how to setup:
+ * poor man's OOP but
+ * allows type to be changed after constructor:
+ Because can't guarantee that ``@transform`` ``@merge`` is the first Ruffus decorator to be encountered.
+ * ``setup_task_func()`` is called for every task before pipeline_xxx()
+ #. Much more informative messages for errors when parsing decorator arguments
+ #. Pipeline decorator methods renamed to decorator_xxx as in ``decorator_follows``
+ #. ``Task.get_task_name()``
+ * rename to ``Task.get_display_name()``
+ * distinguish between decorator and OO interface
+ #. Rename ``_task`` to ``Task``
+ #. Identifying tasks from t_job_result:
+ * job results do not contain references to ``Task`` so that it can be marshalled more easily
+ * we need to look up task at job completion
+ * use ``_node_index`` from ``graph.py`` so we have always a unique identifier for each ``Task``
+ #. Parse arguments using ruffus_utility.parse_task_arguments
+ * Reveals full hackiness and inconsistency between ``add_inputs`` and ``inputs``. The latter only takes a single argument. Each of the elements of the former gets added along side the existing inputs.
+ #. Add ``Pipeline`` class
+ * Create global called ``"main"`` (accessed by Pipeline.pipelines["main"])
+ #. Task name lookup
+ * Task names are unique (Otherwise Ruffus will complain at Task creation)
+ * Can also lookup by fully qualified or unqualified function name but these can be ambiguous
+ * Ambiguous lookups give a list of tasks only so we can have nice diagnostic messages ... UI trumps clean design
+ #. Look up strings across pipelines
+ #. Is pipeline name qualified? Check that
+ #. Check default (current) pipeline
+ #. Check if pipeline name. In which case returns all tail functions
+ #. Check all pipelines
+
+ * Will blow up at any instance of ambiguity in any particular pipeline
+ * Will blow up at any instance of ambiguity across pipelines
+ * Note that mis-spellings will cause problems but if this were c++, I would enforce stricter checking
+ #. Look up functions across pipelines
+ * Try current pipeline first, then all pipelines
+ * Will blow up at any instance of ambiguity in any particular pipeline
+ * Will blow up at any instance of ambiguity across pipelines (if not in current pipeline)
+ #. @mkdir, @follows(mkdir)
+ #. ``Pipeline.get_head_tasks(self)`` (including tasks with mkdir())
+ #. ``Pipeline.get_tail_tasks(self)``
+ #. ``Pipeline._complete_task_setup()`` which follows chain of dependencies for each task in a pipeline
+
+
+======================================================================================================
+Pipeline and Task creation
+======================================================================================================
+
+ * Share code as far as possible between decorator and OOP syntax
+ * Cannot use textbook OOP inheritance hierarchy easily because @decorators are not necessarily
+ given in order.
+
+
+ .. <<python
+
+ .. code-block:: python
+
+ Pipeline.transform
+ _do_create_task_by_OOP()
+
+ @transform
+ Pipeline._create_task()
+ task._decorator_transform
+
+ task._prepare_transform()
+ self.setup_task_func = self._transform_setup
+ parse_task_arguments
+
+
+ Pipeline.run
+ pipeline._complete_task_setup()
+ # walk up ancestors of all task and call setup_task_func
+ unprocessed_tasks = Pipeline.tasks
+ while len(unprocessed_tasks):
+ ancestral_tasks = setup_task_func()
+ if not already processed:
+ unprocessed_tasks.append(ancestral_tasks)
+
+ Call _complete_task_setup() for all the pipelines of each task
+
+ ..
+ python
+
+
+======================================================================================================
+Connecting Task into a DAG
+======================================================================================================
+
+ .. <<python
+
+ ::
+
+ task._complete_setup()
+ task._remove_all_parents()
+ task._deferred_connect_parents()
+ task._setup_task_func()
+ task._handle_tasks_globs_in_inputs()
+ task._connect_parents()
+ # re-lookup task from names in current pipeline so that pipeline.clone() works
+
+ ..
+ python
+
+ * Task dependencies are normally deferred and saved to ``Task.deferred_follow_params``
+ * If Task dependencies call for a new Task (``follows``/``follows(mkdir)``), this takes place
+ immediately
+ * The parameters in ``Task.deferred_follow_params`` are updated with the created ``Task`` when
+ this happens
+ * ``Task._prepare_preceding_mkdir()`` has a ``defer`` flag to prevent it from updating
+ ``Task.deferred_follow_params`` when it is called to resolve deferred dependencies from
+ ``Task._connect_parents()``. Otherwise we will have two copies of each deferred dependency...
+ * ``Task.deferred_follow_params`` must be deep-copied otherwise cloned pipelines will interfere
+ with each other when dependencies are resolved...
+
+
+
+
+
+
+
+
+
diff --git a/doc/installation.rst b/doc/installation.rst
index f3ca116..618f53d 100644
--- a/doc/installation.rst
+++ b/doc/installation.rst
@@ -7,9 +7,15 @@ Installation
:mod:`Ruffus` is a lightweight python module for building computational pipelines.
+.. note ::
+ Ruffus requires Python 2.6 or higher or Python 3.0 or higher
+
+
+
+==============================================================================
The easy way
-============
+==============================================================================
*Ruffus* is available as an
`easy-install <http://peak.telecommunity.com/DevCenter/EasyInstall>`_ -able package
@@ -19,20 +25,17 @@ The easy way
sudo pip install ruffus --upgrade
- This may also work for older installations
+ This may also work for older installations::
- #) Install setuptools::
-
- wget peak.telecommunity.com/dist/ez_setup.py
- sudo python ez_setup.py
+ easy_install -U ruffus
- #) Install *Ruffus* automatically::
- easy_install -U ruffus
+ See below if ``eady_install`` is missing
+==============================================================================
The most up-to-date code:
-==============================
+==============================================================================
* `Download the latest sources <https://pypi.python.org/pypi/ruffus>`_ or
* Check out the latest code from Google using git::
@@ -48,19 +51,49 @@ The most up-to-date code:
python ./setup.py install
-
======================
-Graphical flowcharts
+Prequisites
======================
+==============================================================================
+Installing easy_install
+==============================================================================
+
+ If your system doesn't have ``easy_install``, you can `install <ubuntu/linux mint>`__ one using a package manager, for example::
+
+ # ubuntu/linux mint
+ $ sudo apt-get install python-setuptools
+ $ or sudo yum install python-setuptools
+
+ or manually::
+
+ sudo curl http://peak.telecommunity.com/dist/ez_setup.py | python
+
+ or manually::
+
+ wget peak.telecommunity.com/dist/ez_setup.py
+ sudo python ez_setup.py
+
+
+
+==============================================================================
+Installing pip
+==============================================================================
+
+ If Pip is missing::
+
+ $ sudo easy_install -U pip
+
+
+==============================================================================
+ Graphical flowcharts The most up-to-date code:
+==============================================================================
**Ruffus** relies on the ``dot`` programme from `Graphviz <http://www.graphviz.org/>`_
("Graph visualisation") to make pretty flowchart representations of your pipelines in multiple
graphical formats (e.g. ``png``, ``jpg``). The crossplatform Graphviz package can be
`downloaded here <http://www.graphviz.org/Download.php>`_ for Windows,
- Linux, Macs and Solaris. Some Linux
- distributions may include prebuilt packages.
- For Fedora, try
+ Linux, Macs and Solaris. For Fedora, try
::
yum list 'graphviz*'
diff --git a/doc/static_data/example_scripts/play_with_colours.py b/doc/static_data/example_scripts/play_with_colours.py
index 5054aa4..db16921 100644
--- a/doc/static_data/example_scripts/play_with_colours.py
+++ b/doc/static_data/example_scripts/play_with_colours.py
@@ -196,7 +196,7 @@ custom_flow_chart_colour_scheme["colour_scheme_index"] = options.colour_scheme_i
#
# Overriding colours
#
-if options.colour_scheme_index == None:
+if options.colour_scheme_index is None:
custom_flow_chart_colour_scheme["Vicious cycle"]["linecolor"] = '"#FF3232"'
custom_flow_chart_colour_scheme["Pipeline"]["fontcolor"] = '"#FF3232"'
custom_flow_chart_colour_scheme["Key"]["fontcolor"] = "black"
diff --git a/doc/static_data/example_scripts/simpler.py b/doc/static_data/example_scripts/simpler.py
index c858130..ba6501d 100644
--- a/doc/static_data/example_scripts/simpler.py
+++ b/doc/static_data/example_scripts/simpler.py
@@ -8,7 +8,7 @@
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-# options
+# options
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -33,18 +33,18 @@ parser = OptionParser(version="%prog 1.0")
parser.add_option("-t", "--target_tasks", dest="target_tasks",
action="append",
default = list(),
- metavar="JOBNAME",
+ metavar="JOBNAME",
type="string",
help="Target task(s) of pipeline.")
parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
action="append",
default = list(),
- metavar="JOBNAME",
+ metavar="JOBNAME",
type="string",
help="Pipeline task(s) which will be included even if they are up to date.")
parser.add_option("-j", "--jobs", dest="jobs",
default=5,
- metavar="jobs",
+ metavar="jobs",
type="int",
help="Specifies the number of jobs (commands) to run simultaneously.")
parser.add_option("-v", "--verbose", dest = "verbose",
@@ -52,12 +52,12 @@ parser.add_option("-v", "--verbose", dest = "verbose",
help="Do not echo to shell but only print to log.")
parser.add_option("-d", "--dependency", dest="dependency_file",
default="simple.svg",
- metavar="FILE",
+ metavar="FILE",
type="string",
help="Print a dependency graph of the pipeline that would be executed "
"to FILE, but do not execute it.")
parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
- metavar="FORMAT",
+ metavar="FORMAT",
type="string",
default = 'svg',
help="format of dependency graph file. Can be 'ps' (PostScript), "+
@@ -78,7 +78,7 @@ parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
action="store_true", default=False,
help="Draw horizontal dependency graph.")
-parameters = [
+parameters = [
]
@@ -89,7 +89,7 @@ parameters = [
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-# imports
+# imports
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -119,16 +119,16 @@ def create_custom_file_func(params):
for job_param in params:
yield job_param
return cust_func
-
-
+
+
def is_job_uptodate (infiles, outfiles, *extra_params):
"""
assumes first two parameters are files, checks if they are up to date
"""
return task.needs_update_check_modify_time (infiles, outfiles, *extra_params)
-
-
-
+
+
+
def test_post_task_function ():
print "Hooray"
@@ -142,11 +142,11 @@ def test_job_io(infiles, outfiles, extra_params):
params = (infiles, outfiles) + extra_params
sys.stdout.write(' job = %s\n' % json.dumps(params))
-
-
+
+
if isinstance(infiles, str):
infiles = [infiles]
- elif infiles == None:
+ elif infiles is None:
infiles = []
if isinstance(outfiles, str):
outfiles = [outfiles]
@@ -240,21 +240,21 @@ def task4(infiles, outfiles, *extra_params):
-
+
if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
- long_winded=True,
+ pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
+ long_winded=True,
gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
elif options.dependency_file:
pipeline_printout_graph ( open(options.dependency_file, "w"),
options.dependency_graph_format,
- options.target_tasks,
+ options.target_tasks,
options.forced_tasks,
draw_vertically = not options.draw_horizontally,
gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
no_key_legend = options.no_key_legend_in_graph)
-else:
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
+else:
+ pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
-
+
diff --git a/doc/static_data/ruffus.pdf b/doc/static_data/ruffus.pdf
index 68836b8..0c4877e 100644
Binary files a/doc/static_data/ruffus.pdf and b/doc/static_data/ruffus.pdf differ
diff --git a/doc/static_data/small_logo.png b/doc/static_data/small_logo.png
new file mode 100644
index 0000000..77e99c0
Binary files /dev/null and b/doc/static_data/small_logo.png differ
diff --git a/doc/todo.rst b/doc/todo.rst
index 678f521..36c5320 100644
--- a/doc/todo.rst
+++ b/doc/todo.rst
@@ -3,25 +3,85 @@
.. _todo:
##########################################
-Future Changes to Ruffus
+Where I see Ruffus going
##########################################
- I would appreciated feedback and help on all these issues and where next to take *ruffus*.
+ These are the future enhancements I would like to see in Ruffus:
+ * Simpler syntax
+ * Extremely pared down syntax where strings are interpreted as commands (like gmake)
+ but with full Ruffus support / string interpolation etc.
+ * More powerful non-decorator OOP syntax
+ * More customisation points for your own syntax / database use
- **Future Changes** are features where we more or less know where we are going and how to get there.
+ * Better support for Computational clusters / larger scale pipelines
+ * Running jobs out of sequence
+ * Long running pipeline where input can be added later
+ * Restarting failed jobs robustly
+ * Finding out why jobs fail
+ * Does Ruffus scale to thousands of parallel jobs. What are the bottlenecks?
- **Planned Improvements** describes features we would like in Ruffus but where the implementation
- or syntax has not yet been (fully) worked out.
+ * Better displays of progress
+ * Query which tasks / jobs are being run
+ * GUI displays
- If you have suggestions or contributions, please either write to me ( ruffus_lib at llew.org.uk) or
- send a pull request via the `git site <https://github.com/bunbun/ruffus>`__.
+ * Dynamic control during pipeline progress
+ * Turn tasks on and off
+ * Pause pipelines
+ * Pause jobs
+ * Change priorities
+
+ * Better handling of data
+ * Can we read and write from databases instead of files?
+ * Can we cleanup files but preserve history?
+
+##########################################
+In up coming release:
+##########################################
+
+****************************************************************************************
+Todo: document ``output_from()``
+****************************************************************************************
+
+****************************************************************************************
+Todo: document new syntax
+****************************************************************************************
+
+****************************************************************************************
+Todo: Log the progress through the pipeline in a machine parsable format
+****************************************************************************************
+
+ Standard parsable format for reporting the state of the pipeline enhancement
+
+ * Timestamped text file
+ * Timestamped Database
+
+****************************************************************************************
+Todo: either_or: Prevent failed jobs from propagating further
+****************************************************************************************
+
+ Motivating example:
+
+ .. <<Python_
+
+ .. code-block:: python
+
+ @transform(prevtask, suffix(".txt"), either_or(".failed", ".succeed"))
+ def task(input_file, output_files):
+ succeed_file_name, failed_file_name = output_files
+ if not run_operation(input_file, succeed_file_name):
+ # touch failed file
+ with open(failed_file_name, "w") as faile_file:
+ pass
+
+ ..
+ Python_
.. _todo.inactive_tasks_in_pipeline_printout_graph:
********************************************************************************************************
-Todo: pipeline_printout_graph should print inactive tasks
+Todo: (bug fix) pipeline_printout_graph should print inactive tasks
********************************************************************************************************
@@ -58,6 +118,34 @@ Todo: Mark input strings as non-file names, and add support for dynamically retu
Check!
+
+##########################################
+Future Changes to Ruffus
+##########################################
+
+ I would appreciated feedback and help on all these issues and where next to take *ruffus*.
+
+
+ **Future Changes** are features where we more or less know where we are going and how to get there.
+
+ **Planned Improvements** describes features we would like in Ruffus but where the implementation
+ or syntax has not yet been (fully) worked out.
+
+ If you have suggestions or contributions, please either write to me ( ruffus_lib at llew.org.uk) or
+ send a pull request via the `git site <https://github.com/bunbun/ruffus>`__.
+
+
+
+****************************************************************************************
+Todo: Replacements for formatter(), suffix(), regex()
+****************************************************************************************
+
+ formatter etc. should be self contained objects derived from a single base class
+ with behaviour rather than empty tags used for dispatching to functions
+
+ The design is better fit by and should be switched over to an inheritance scheme
+
+
.. _todo.extra_parameters:
********************************************************************************************************
@@ -186,10 +274,6 @@ Todo: ``@recombine``
This is the only way job trickling can work without stalling the pipeline: We would know
how many jobs were pending for each ``@recombine`` job and which jobs go together.
-****************************************************************************************
-Todo: Named parameters in decorators for clarity
-****************************************************************************************
-
.. _todo.bioinformatics_example:
********************************************************************************************************
@@ -258,6 +342,50 @@ Implementation
* When all slots are full in each job, this triggers putting the job parameters onto the job submission queue
* The pipeline object should allow Ruffus to be reentrant?
+.. _todo.move_checkpoint_files:
+
+********************************************************************************************************
+Todo: Allow checkpoint files to be moved
+********************************************************************************************************
+
+ Allow checkpoint files to be "rebased" so that moving the working directory of the pipeline does not
+ invalidate all the files.
+
+ We need some sort of path search and replace mechanism which handles conflicts, and probably versioning?
+
+
+
+.. _todo.intermediate_files:
+
+********************************************************************************************************
+Todo: Remove intermediate files
+********************************************************************************************************
+
+ Often large intermediate files are produced in the middle of a pipeline which could be
+ removed. However, their absence would cause the pipeline to appear out of date. What is
+ the best way to solve this?
+
+ In gmake, all intermediate files which are not marked ``.PRECIOUS`` are deleted.
+
+ We can similar mark out all tasks producing intermediate files so that all their output file can be deleted using an ``@intermediate/provisional/transient/temporary/interim/ephemeral`` decorator.
+
+ The tricky part of the design is how to delete files without disrupting our ability to build the original file dependency DAG, and hence
+ check which tasks have up-to-date output when the pipeline is run again.
+
+ 1. We can just work back from upstream/downstream files and ignore the intermediate files as gmake does. However,
+ the increased power of Ruffus makes this very fragile: In gmake, the DAG is entirely specified by the specified destination files.
+ In Ruffus, the number of task files is indeterminate, and can be changed at run time (see @split and @subdivide)
+ 2. We can save the filenames into the checksum file before deleting them
+ 3. We can leave the files in place files but zero out their contents. It is probably best
+ to write a small magic text value to the file, e.g. "RUFFUS_ZEROED_FILE", so that we are
+ not confused by real files of zero size.
+
+ In practice (2) and (3) should be combined for safety.
+
+ 1. pipeline_cleaunup() will print out a list of files to be zeroed, or a list of commands to zero files or just do it for you
+ 2. When rerunning, we can force files to be recreated using ``pipeline_run(..., forcedtorun_tasks,...)``, and Ruffus will track back
+ through lists of dependencies and recreate all "zeroed" files.
+
##########################################
@@ -295,6 +423,7 @@ Planned: Running python code (task functions) transparently on remote cluster no
* exception
#) Full version use libpythongrid?
+ * http://zguide.zeromq.org/page:all
* Christian Widmer <ckwidmer at gmail.com>
* Cheng Soon Ong <chengsoon.ong at unimelb.edu.au>
* https://code.google.com/p/pythongrid/source/browse/#git%2Fpythongrid
@@ -364,79 +493,6 @@ Planned: Non-decorator / Function interface to Ruffus
********************************************************************************************************
-.. _todo.intermediate_files:
-
-********************************************************************************************************
-Planned: Remove intermediate files
-********************************************************************************************************
-
- Often large intermediate files are produced in the middle of a pipeline which could be
- removed. However, their absence would cause the pipeline to appear out of date. What is
- the best way to solve this?
-
- In gmake, all intermediate files which are not marked ``.PRECIOUS`` are deleted.
-
- We do not want to manually mark intermediate files for several reasons:
- * The syntax would be horrible and clunky
- * The gmake distinction between ``implicit`` and ``explicit`` rules is not one we
- would like to impose on Ruffus
- * Gmake uses statically determined (DAG) dependency trees so it is quite natural and
- easy to prune intermediate paths
-
- Our preferred solution should impose little to no semantic load on Ruffus, i.e. it should
- not make it more complex / difficult to use. There are several alternatives we are
- considering:
-
- #) Have an **update** mode in which pipeline_run would ignore missing files and only run tasks with existing, out-of-date files.
- #) Optionally ignore all out-of-date dependencies beyond a specified point in the pipeline
- #) Add a decorator to flag sections of the pipeline where intermediate files can be removed
-
-
- Option (1) is rather unnerving because it makes inadvertent errors difficult to detect.
-
- Option (2) involves relying on the user of a script to remember the corect chain of dependencies in
- often complicated pipelines. It would be advised to keep a flowchart to hand. Again,
- the chances of error are much greater.
-
- Option (3) springs from the observation by Andreas Heger that parts of a pipeline with
- disposable intermediate files can usually be encapsulated as an autonomous section.
- Within this subpipeline, all is well provided that the outputs of the last task are complete
- and up-to-date with reference to the inputs of the first task. Intermediate files
- could be removed with impunity.
-
- The suggestion is that these autonomous subpipelines could be marked out using the Ruffus
- decorator syntax::
-
- #
- # First task in autonomous subpipeline
- #
- @files("who.isit", "its.me")
- def first_task(*args):
- pass
-
- #
- # Several intermediate tasks
- #
- @transform(subpipeline_task1, suffix(".me"), ".her")
- def task2_etc(*args):
- pass
-
- #
- # Final task
- #
- @sub_pipeline(subpipeline_task1)
- @transform(subpipeline_task1, suffix(".her"), ".you")
- def final_task(*args):
- pass
-
- **@sub_pipeline** marks out all tasks between ``first_task`` and ``final_task`` and
- intermediate files such as ``"its.me"``, ``"its.her`` can be deleted. The pipeline will
- only run if ``"its.you"`` is missing or out-of-date compared with ``"who.isit"``.
-
- Over the next few Ruffus releases we will see if this is a good design, and whether
- better keyword can be found than **@sub_pipeline** (candidates include **@shortcut**
- and **@intermediate**)
-
.. _todo.retry:
diff --git a/doc/tutorials/new_syntax.rst b/doc/tutorials/new_syntax.rst
new file mode 100644
index 0000000..688bd1e
--- /dev/null
+++ b/doc/tutorials/new_syntax.rst
@@ -0,0 +1,485 @@
+.. include:: ../global.inc
+
+.. role:: raw-html(raw)
+ :format: html
+
+:raw-html:`<style> .highlight-blue {color:blue} </style>`
+
+:raw-html:`<style> .highlight-red {color:red} </style>`
+
+.. role:: highlight-red
+
+.. role:: highlight-blue
+
+.. :highlight-red:`Test.`
+
+.. _new_syntax:
+
+
+################################################################################
+New Object orientated syntax for Ruffus in Version 2.6
+################################################################################
+
+ Ruffus Pipelines can now be created and manipulated directly using :highlight-red:`Pipeline` and :highlight-red:`Task` objects instead of via decorators.
+
+.. note::
+
+ You may want to go through the :ref:`worked_example <new_syntax.worked_example>` first.
+
+
+==============================================================================
+Syntax
+==============================================================================
+
+
+ This traditional Ruffus code:
+
+ .. <<python
+
+ .. code-block:: python
+
+ from ruffus import *
+
+ # task function
+ starting_files = ["input/a.fasta","input/b.fasta"]
+ @transform(input = starting_files,
+ filter = suffix('.fasta'),
+ output = '.sam',
+ output_dir = "output")
+ def map_dna_sequence(input_file, output_file) :
+ pass
+
+ pipeline_run()
+
+
+ ..
+ python
+
+
+ Can also be written as:
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 9
+
+ from ruffus import *
+
+ # undecorated task function
+ def map_dna_sequence(input_file, output_file) :
+ pass
+
+ starting_files = ["input/a.fasta","input/b.fasta"]
+
+ # make ruffus Pipeline() object
+ my_pipeline = Pipeline(name = "test")
+ my_pipeline.transform(task_func = map_dna_sequence,
+ input = starting_files,
+ filter = suffix('.fasta'),
+ output = '.sam',
+ output_dir = "output")
+
+ my_pipeline.run()
+ ..
+ python
+
+ | The two different syntax are almost identical:
+ | The first parameter **task_func=**\ ``your_python_function`` is mandatory.
+ | Otherwise, all other parameters are in the same order as before, and can be given by position or as named arguments.
+
+==============================================================================
+Advantages
+==============================================================================
+
+ These are some of the advantages of the new syntax:
+
+ #) **Pipeline topology is assembled in one place**
+
+ This is a matter of personal preference.
+
+ Nevertheless, using decorators to locally annotate python functions with pipeline parameters arguably
+ helps separation of concerns.
+
+ #) **Pipelines can be created** *on the fly*
+
+ For example, using parameters parsed from configuration files.
+
+ Ruffus pipelines no longer have to be defined at global scope.
+
+ #) **Reuse common sub-pipelines**
+
+ Shared sub pipelines can be created from discrete python modules and joined together
+ as needed. Bioinformaticists may have "mapping", "aligning", "variant-calling" sub-pipelines etc.
+
+ #) **Multiple Tasks can share the same python function**
+
+ Tasks are normally referred to by their associated functions (as with decoratored Ruffus tasks).
+ However, you can also disambiguate Tasks by specifying their name directly.
+
+ #) **Pipeline topology can be specified at run time**
+
+ Some (especially bioinformatics) tasks require binary merging. This can be very inconvenient.
+
+ For example, if we have 8 data files, we need three successive rounds of merging (8->4->2->1)
+ or three tasks) to produce the output. But if we are given 10 data files, we now find that
+ we needed to have four tasks for four rounds of merging (10->5->3->2->1).
+
+ There was previously no easy way to arrange different Ruffus topologies in response to the
+ data. Now we can add as many extra merging tasks to our pipeline (all sharing the same underlying
+ python function) as needed.
+
+
+
+==============================================================================
+Compatability
+==============================================================================
+
+ * The changes are fully backwards compatibile. All valid Ruffus code continues to work
+ * Decorators and ``Pipeline`` objects can be used interchangeably:
+
+ Decorated functions are automatically part of a default constructed ``Pipeline`` named ``"main"``.
+ .. code-block:: python
+
+ main_pipeline = Pipeline.pipelines["main"]
+
+ ..
+
+ In the following example, a pipeline using the
+ Ruffus with classes syntax :highlight-red:`(1)` and :highlight-red:`(3)` has a traditionally decorated task function in the middle :highlight-red:`(2)`.
+
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 15, 21, 32
+
+ from ruffus import *
+
+ # get default pipeline
+ main_pipeline = Pipeline.pipelines["main"]
+
+ # undecorated task functions
+ def compress_sam_to_bam(input_file, output_file) :
+ open(output_file, "w").close()
+
+ def create_files(output_file) :
+ open(output_file, "w").close()
+
+
+ #
+ # 1. Ruffus with classes
+ #
+ starting_files = main_pipeline.originate(create_files, ["input/a.fasta","input/b.fasta"])\
+ .follows(mkdir("input", "output"))
+
+ #
+ # 2. Ruffus with python decorations
+ #
+ @transform(starting_files,
+ suffix('.fasta'),
+ '.sam',
+ output_dir = "output")
+ def map_dna_sequence(input_file, output_file) :
+ open(output_file, "w").close()
+
+
+ #
+ # 3. Ruffus with classes
+ #
+ main_pipeline.transform(task_func = compress_sam_to_bam,
+ input = map_dna_sequence,
+ filter = suffix(".sam"),
+ output = ".bam")
+
+ # main_pipeline.run()
+ # or
+ pipeline_run()
+
+
+ ..
+ python
+
+
+==============================================================================
+Class methods
+==============================================================================
+
+ The **ruffus.Pipeline** class has the following self-explanatory methods:
+
+ .. <<python
+
+ .. code-block:: python
+
+ Pipeline.run(...)
+ Pipeline.printout(...)
+ Pipeline.printout_graph(...)
+
+ ..
+ python
+
+
+ These methods return a **ruffus.Task** object
+
+ .. <<python
+
+
+ .. code-block:: python
+
+ Pipeline.originate(...)
+ Pipeline.transform(...)
+ Pipeline.split(...)
+ Pipeline.merge(...)
+ Pipeline.mkdir(...)
+
+ Pipeline.collate(...)
+ Pipeline.subdivide(...)
+
+ Pipeline.combinations(...)
+ Pipeline.combinations_with_replacement(...)
+ Pipeline.product(...)
+ Pipeline.permutations(...)
+
+ Pipeline.follows(...)
+ Pipeline.check_if_uptodate(...)
+ Pipeline.graphviz(...)
+
+ Pipeline.files(...)
+ Pipeline.parallel(...)
+
+ ..
+ python
+
+
+ A Ruffus **Task** can be modified with the following methods
+
+ .. <<python
+
+ .. code-block:: python
+
+ Task.active_if(...)
+ Task.check_if_uptodate(...)
+ Task.follows(...)
+ Task.graphviz(...)
+ Task.jobs_limit(...)
+ Task.mkdir(...)
+ Task.posttask(...)
+
+ ..
+ python
+
+==============================================================================
+Call chaining
+==============================================================================
+
+ The syntax is designed to allow call chaining:
+
+ .. <<python
+
+ .. code-block:: python
+
+ Pipeline.transform(...)\
+ .mkdir(follows(...))\
+ .active_if(...)\
+ .graphviz(...)
+
+ ..
+ python
+
+
+==============================================================================
+Referring to Tasks
+==============================================================================
+
+ Ruffus pipelines are chained together or specified by referring to each stage or Task.
+
+ :highlight-red:`(1)` and :highlight-red:`(2)` are ways to referring to tasks that Ruffus has always supported.
+
+ :highlight-red:`(3)` - :highlight-red:`(6)` are new to Ruffus v 2.6 but apply
+ to both using decorators or the new Ruffus with classes syntax.
+
+______________________________________________________________________________
+1) Python function
+______________________________________________________________________________
+
+ .. <<python
+
+ .. code-block:: python
+
+ @transform(prev_task, ...)
+ def next_task():
+ pass
+
+ pipeline.transform(input = next_task, ...)
+
+ ..
+ python
+
+______________________________________________________________________________
+2) Python function name (using *output_from*)
+______________________________________________________________________________
+
+
+ .. <<python
+
+ .. code-block:: python
+
+ pipeline.transform(input = output_from("prev_task"), ...)
+
+ ..
+ python
+
+.. note::
+
+ The above :highlight-red:`(1) and (2) only work if the Python function specifies the task unambiguously in a pipeline.`
+ If you reuse the same Python function for multiple tasks, use the following methods.
+
+ Ruffus will complain with Exceptions if your code is ambiguous.
+
+
+______________________________________________________________________________
+3) Task object
+______________________________________________________________________________
+
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 3
+
+ prev_task = pipeline.transform(...)
+
+ # prev_task is a Task object
+ next_task = pipeline.transform(input = prev_task, ....)
+
+ ..
+ python
+
+______________________________________________________________________________
+4) Task name (using *output_from*)
+______________________________________________________________________________
+
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 1
+
+ # name this task "prev_task"
+ pipeline.transform(name = "prev_task",...)
+
+ pipeline.transform(input = output_from("prev_task"), ....)
+
+ ..
+ python
+
+ .. note::
+
+ Tasks from other pipelines can be referred to using full qualified names in the **pipeline**::*task* format
+
+ .. <<python
+
+ .. code-block:: python
+
+ pipeline.transform(input = output_from("other_pipeline::prev_task"), ....)
+
+ ..
+ python
+
+
+______________________________________________________________________________
+5) Pipeline
+______________________________________________________________________________
+
+
+ When we are assembling our pipeline from sub-pipelines (especially those in other modules which other people might have written)
+ it is inconvenient to break encapsulation to find out the component **Task** of the subpipeline.
+
+ In which case, the sub-pipeline author can assign particular tasks to the **head** and **tail** of the pipeline.
+ The pipeline will be an alias for these:
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 5,8
+
+ # Note: these functions take lists
+ sub_pipeline.set_head_tasks([first_task])
+ sub_pipeline.set_tail_tasks([last_task])
+
+ # first_task.set_input(...)
+ sub_pipeline.set_input(input = "*.txt")
+
+ # (input = last_task,...)
+ main_pipeline.transform(input = sub_pipeline, ....)
+
+ ..
+ python
+
+
+ If you don't have access to a pipeline object, you can look it up via the Pipeline object
+
+ .. code-block:: python
+ :emphasize-lines: 1
+
+ # This is the default "main" pipeline which holds decorated task functions
+ main_pipeline = Pipeline.pipelines["main"]
+
+ my_pipeline = Pipeline("test")
+
+ alias_to_my_pipeline = Pipeline.pipelines["test"]
+
+ ..
+ python
+
+
+
+______________________________________________________________________________
+6) Lookup Task via the Pipeline
+______________________________________________________________________________
+
+ We can ask a Pipeline to lookup task names, functions and function names for us.
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 1,4,7
+
+ # Lookup task name
+ pipeline.transform(input = pipeline["prev_task"], ....)
+
+ # Lookup via python function
+ pipeline.transform(input = pipeline[python_function], ....)
+
+ # Lookup via python function name
+ pipeline.transform(input = pipeline["python_function_name"], ....)
+
+ ..
+ python
+
+ This is straightforward if the lookup is unambiguous for the pipeline.
+
+ If the names are not found in the pipeline, Ruffus will look across all pipelines.
+
+ Any ambiguities will result in an immediate error.
+
+ *In extremis*, you can use pipeline qualified names
+
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 1
+
+ # Pipeline qualified task name
+ pipeline.transform(input = pipeline["other_pipeline::prev_task"], ....)
+
+ ..
+ python
+
+
+
+.. note::
+
+ All this will be much clearer going through the :ref:`worked_example <new_syntax.worked_example>`.
+
+
diff --git a/doc/tutorials/new_syntax_worked_example.rst b/doc/tutorials/new_syntax_worked_example.rst
new file mode 100644
index 0000000..5ae682b
--- /dev/null
+++ b/doc/tutorials/new_syntax_worked_example.rst
@@ -0,0 +1,355 @@
+.. include:: ../global.inc
+
+.. role:: raw-html(raw)
+ :format: html
+
+:raw-html:`<style> .highlight-blue {color:blue} </style>`
+
+:raw-html:`<style> .highlight-red {color:red} </style>`
+
+.. role:: highlight-red
+
+.. role:: highlight-blue
+
+.. :highlight-red:`Test.`
+
+.. _new_syntax.worked_example:
+
+
+################################################################################
+Worked Example for New Object orientated syntax for Ruffus in Version 2.6
+################################################################################
+
+ Ruffus Pipelines can now be created and manipulated directly using :highlight-red:`Pipeline` and :highlight-red:`Task` objects instead of via decorators.
+
+ For clarity, we use named parameters in this example. You can just as easily pass all parameters by position.
+
+==============================================================================
+Worked example
+==============================================================================
+
+ .. note::
+
+ Remember to look at the example code:
+
+ * :ref:`new_syntax.worked_example.code`
+
+ This example pipeline is a composite of three separately subpipelines each created by a python
+ function ``make_pipeline1()`` which is joined to another subpipeline created by ``make_pipeline2()``
+
+ .. image:: ../images/subpipeline_example.png
+ :scale: 50
+
+
+ Although there are 13 different stages to this pipeline, we are using the same three python functions
+ (but supplying them with different data).
+
+
+ .. <<python
+
+ .. code-block:: python
+
+ def task_originate(o):
+ # Makes new files
+ ...
+
+ def task_m_to_1(i, o):
+ # Merges files together
+ ...
+
+ def task_1_to_1(i, o):
+ # One input per output
+ ...
+
+ ..
+ python
+
+______________________________________________________________________________
+Pipeline factory
+______________________________________________________________________________
+
+
+ Let us start with a python function which makes a full formed sub pipeline useable as a modular building block
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 1
+
+ # Pipelines need to have a unique name
+ def make_pipeline1(pipeline_name,
+ starting_file_names):
+ pass
+
+ ..
+ python
+
+
+ Note that we are passing the pipeline name as the first parameter.
+
+ All pipelines must have unique names
+
+
+ .. <<python
+
+ .. code-block:: python
+
+ test_pipeline = Pipeline(pipeline_name)
+
+ new_task = test_pipeline.originate(task_func = task_originate,
+ output = starting_file_names)\
+ .follows(mkdir(tempdir), mkdir(tempdir + "testdir", tempdir + "testdir2"))\
+ .posttask(touch_file(tempdir + "testdir/whatever.txt"))
+
+ ..
+ python
+
+ A new task is returned from ``test_pipeline.originate(...)`` which is then modified via ``.follows(...)``
+ and ``.posttask(...)``. This is familiar Ruffus syntax only slightly rearranged.
+
+ We can change the ``output=starting_file_names`` later using ``set_output()`` but sometimes it
+ is just more convenient to pass this as a parameter to the pipeline factory function.
+
+
+ .. note::
+
+ The first, mandatory parameter is ``task_func = task_originate`` which is the python function for this task
+
+______________________________________________________________________________
+Three different ways of referring to input Tasks
+______________________________________________________________________________
+
+ Just as in traditional Ruffus, Pipelines are created by setting the **input** of one task to (the **output** of) its predecessor.
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 3,11,12,18
+
+ test_pipeline.transform(task_func = task_m_to_1,
+ name = "add_input",
+ # Lookup Task from function task_originate()
+ # Needs to be unique in the pipeline
+ input = task_originate,
+ filter = regex(r"(.*)"),
+ add_inputs = add_inputs(tempdir + "testdir/whatever.txt"),
+ output = r"\1.22")
+ test_pipeline.transform(task_func = task_1_to_1,
+ name = "22_to_33",
+ # Lookup Task from unique Task name = "add_input"
+ # Function name is not unique in the pipeline
+ input = output_from("add_input"),
+ filter = suffix(".22"),
+ output = ".33")
+ tail_task = test_pipeline.transform(task_func = task_1_to_1,
+ name = "33_to_44",
+ # Ask test_pipeline to lookup Task name = "22_to_33"
+ input = test_pipeline["22_to_33"],
+ filter = suffix(".33"),
+ output = ".44")
+
+ ..
+ python
+
+
+______________________________________________________________________________
+Head and Tail Tasks
+______________________________________________________________________________
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 1,7
+
+ # Set the tail task: test_pipeline can be used as an input
+ # without knowing the details of task names
+ #
+ # Use Task object=tail_task directly
+ test_pipeline.set_tail_tasks([tail_task])
+
+ # Set the head task: we can feed input into test_pipeline
+ # without knowing the details of task names
+ test_pipeline.set_head_tasks([test_pipeline[task_originate]])
+
+ return test_pipeline
+
+ ..
+ python
+
+
+ By calling ``set_tail_tasks`` and ``set_head_tasks`` to assign the first and last stages of
+ ``test_pipeline``, we can later use ``test_pipeline`` without knowing its component Tasks.
+
+ The last step is to return the fully formed pipeline instance
+
+
+______________________________________________________________________________
+Another Pipeline factory
+______________________________________________________________________________
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 9
+
+ #
+ # Returns a fully formed sub pipeline useable as a building block
+ #
+ def make_pipeline2( pipeline_name = "pipeline2", do_not_define_head_task = False):
+ test_pipeline2 = Pipeline(pipeline_name)
+ test_pipeline2.transform(task_func = task_1_to_1,
+ # task name
+ name = "44_to_55",
+ # placeholder: will be replaced later with set_input()
+ input = None,
+ filter = suffix(".44"),
+ output = ".55")
+ test_pipeline2.merge( task_func = task_m_to_1,
+ input = test_pipeline2["44_to_55"],
+ output = tempdir + "final.output",)
+ # Lookup task using function name
+ # This is unique within pipeline2
+ test_pipeline2.set_tail_tasks([test_pipeline2[task_m_to_1]])
+
+ # Lookup task using task name
+ test_pipeline2.set_head_tasks([test_pipeline2["44_to_55"]])
+
+ return test_pipeline2
+
+ ..
+ python
+
+
+ make_pipeline2() looks very similar to make_pipeline1 except that the input for the **head** task
+ is left blank for assigning later
+
+ Note that we can use ``task_m_to_1`` to look up a Task (``test_pipeline2[task_m_to_1]``) even
+ though this function is also used by test_pipeline. There is no ambiguity so long as only one
+ task in ``test_pipeline2`` uses this python function.
+
+
+______________________________________________________________________________
+Creating multiple copies of a pipeline
+______________________________________________________________________________
+
+
+ Let us call ``make_pipeline1()`` to make two completely independent pipelines (``"pipeline1a"`` and ``"pipeline1b"``)
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 1
+
+ # First two pipelines are created as separate instances by make_pipeline1()
+ pipeline1a = make_pipeline1(pipeline_name = "pipeline1a", starting_file_names = [tempdir + ss for ss in ("a.1", "b.1")])
+ pipeline1b = make_pipeline1(pipeline_name = "pipeline1b", starting_file_names = [tempdir + ss for ss in ("c.1", "d.1")])
+
+ ..
+ python
+
+ We can also create a new instance of a pipeline by **"cloning"** an existing pipeline
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 1
+
+ # pipeline1c is a clone of pipeline1b
+ pipeline1c = pipeline1b.clone(new_name = "pipeline1c")
+
+ ..
+ python
+
+
+ Because ``"pipeline1c"`` is a clone of ``"pipeline1b"``, it shares exactly the same parameters.
+ Let us change this by giving ``"pipeline1c"`` its own starting files.
+
+ We can do this for normal (e.g. **transform**, **split**, **merge** etc) tasks by calling
+
+ .. <<python
+
+ .. code-block:: python
+
+ transform_task.set_input(input = xxx)
+
+ ..
+ python
+
+ @originate doesn't take **input** but creates results specified in the **output** parameter.
+ To finish setting up ``pipeline1c``:
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 1-2
+
+ # Set the "originate" files for pipeline1c to ("e.1" and "f.1")
+ # Otherwise they would use the original ("c.1", "d.1")
+ pipeline1c.set_output(output = [tempdir + ss for ss in ("e.1", "f.1")])
+
+ ..
+ python
+
+ We only create one copy of ``pipeline2``
+
+ .. <<python
+
+ .. code-block:: python
+
+ pipeline2 = make_pipeline2()
+ ..
+ python
+
+
+
+______________________________________________________________________________
+Connecting pipelines together
+______________________________________________________________________________
+
+
+ Because we have previously assigned **head** and **tail** tasks, we can easily join the pipelines together:
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 3,5
+
+ # Join all pipeline1a-c to pipeline2
+ pipeline2.set_input(input = [pipeline1a, pipeline1b, pipeline1c])
+
+ ..
+ python
+
+
+______________________________________________________________________________
+Running a composite pipeline
+______________________________________________________________________________
+
+
+ Ruffus automatically follows the antecedent dependencies of each task even if they are from another
+ pipeline.
+
+ This means that you can run composite pipelines seamlessly, without any effort:
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 1,4
+
+ # Only runs pipeline1a
+ pipeline1a.run()
+
+ # Runs pipeline1a,b,c -> pipeline2
+ pipeline2.run(multiprocess = 10, verbose = 0)
+
+ ..
+ python
+
+
+
+ .. note::
+
+ Remember to look at the example code:
+
+ * :ref:`new_syntax.worked_example.code`
+
diff --git a/doc/tutorials/new_syntax_worked_example_code.rst b/doc/tutorials/new_syntax_worked_example_code.rst
new file mode 100644
index 0000000..a079615
--- /dev/null
+++ b/doc/tutorials/new_syntax_worked_example_code.rst
@@ -0,0 +1,222 @@
+.. _new_syntax.worked_example.code:
+
+#################################################################################################################
+Python Code for: New Object orientated syntax for Ruffus in Version 2.6
+#################################################################################################################
+
+ .. seealso::
+
+ * :ref:`new_syntax.worked_example <new_syntax.worked_example>`
+
+ This code is from ``test/test_subpipeline.py`` in the **Ruffus** distribution
+
+
+ .. <<python
+
+ .. code-block:: python
+ :emphasize-lines: 70,76-79,86,87,94,95,101,106-107,109,112-114,118,119,125,130,132,150,154,157,158,162
+
+ #!/usr/bin/env python
+ from __future__ import print_function
+ """
+ test_subpipeline.py
+
+ Demonstrates the new Ruffus syntax in version 2.6
+ """
+
+ import os
+ import sys
+
+ # add grandparent to search path for testing
+ grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+ sys.path.insert(0, grandparent_dir)
+
+ import ruffus
+ from ruffus import add_inputs, suffix, mkdir, regex, Pipeline, output_from, touch_file
+ print("\tRuffus Version = ", ruffus.__version__)
+
+ #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+ # imports
+
+
+ #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+ import shutil
+
+
+ def touch (outfile):
+ with open(outfile, "w"):
+ pass
+
+
+ #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+ # Tasks
+
+
+ #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+ tempdir = "tempdir/"
+ def task_originate(o):
+ """
+ Makes new files
+ """
+ touch(o)
+
+ def task_m_to_1(i, o):
+ """
+ Merges files together
+ """
+ with open(o, "w") as o_file:
+ for f in sorted(i):
+ with open(f) as ii:
+ o_file.write(f +"=" + ii.read() + "; ")
+
+ def task_1_to_1(i, o):
+ """
+ 1 to 1 for transform
+ """
+ with open(o, "w") as o_file:
+ with open(i) as ii:
+ o_file.write(i +"+" + ii.read())
+
+ DEBUG_do_not_define_tail_task = False
+ DEBUG_do_not_define_head_task = False
+
+ import unittest
+
+ #
+ # Returns a fully formed sub pipeline useable as a building block
+ #
+ def make_pipeline1(pipeline_name, # Pipelines need to have a unique name
+ starting_file_names):
+ test_pipeline = Pipeline(pipeline_name)
+
+ # We can change the starting files later using
+ # set_input() for transform etc.
+ # or set_output() for originate
+ # But it can be more convenient to just pass this to the function making the pipeline
+ #
+ test_pipeline.originate(task_originate, starting_file_names)\
+ .follows(mkdir(tempdir), mkdir(tempdir + "testdir", tempdir + "testdir2"))\
+ .posttask(touch_file(tempdir + "testdir/whatever.txt"))
+ test_pipeline.transform(task_func = task_m_to_1,
+ name = "add_input",
+ # Lookup Task from function name task_originate()
+ # So long as this is unique in the pipeline
+ input = task_originate,
+ filter = regex(r"(.*)"),
+ add_inputs = add_inputs(tempdir + "testdir/whatever.txt"),
+ output = r"\1.22")
+ test_pipeline.transform(task_func = task_1_to_1,
+ name = "22_to_33",
+ # Lookup Task from Task name
+ # Function name is not unique in the pipeline
+ input = output_from("add_input"),
+ filter = suffix(".22"),
+ output = ".33")
+ tail_task = test_pipeline.transform(task_func = task_1_to_1,
+ name = "33_to_44",
+ # Ask Pipeline to lookup Task from Task name
+ input = test_pipeline["22_to_33"],
+ filter = suffix(".33"),
+ output = ".44")
+
+ # Set the tail task so that users of my sub pipeline can use it as a dependency
+ # without knowing the details of task names
+ #
+ # Use Task() object directly without having to lookup
+ test_pipeline.set_tail_tasks([tail_task])
+
+ # If we try to connect a Pipeline without tail tasks defined, we have to
+ # specify the exact task within the Pipeline.
+ # Otherwise Ruffus will not know which task we mean and throw an exception
+ if DEBUG_do_not_define_tail_task:
+ test_pipeline.set_tail_tasks([])
+
+ # Set the head task so that users of my sub pipeline send input into it
+ # without knowing the details of task names
+ test_pipeline.set_head_tasks([test_pipeline[task_originate]])
+
+ return test_pipeline
+
+ #
+ # Returns a fully formed sub pipeline useable as a building block
+ #
+ def make_pipeline2( pipeline_name = "pipeline2"):
+ test_pipeline2 = Pipeline(pipeline_name)
+ test_pipeline2.transform(task_func = task_1_to_1,
+ # task name
+ name = "44_to_55",
+ # placeholder: will be replaced later with set_input()
+ input = None,
+ filter = suffix(".44"),
+ output = ".55")
+ test_pipeline2.merge( task_func = task_m_to_1,
+ input = test_pipeline2["44_to_55"],
+ output = tempdir + "final.output",)
+
+ # Set head and tail
+ test_pipeline2.set_tail_tasks([test_pipeline2[task_m_to_1]])
+ if not DEBUG_do_not_define_head_task:
+ test_pipeline2.set_head_tasks([test_pipeline2["44_to_55"]])
+
+ return test_pipeline2
+
+
+ def run_pipeline():
+
+ # First two pipelines are created as separate instances by the make_pipeline1 function
+ pipeline1a = make_pipeline1(pipeline_name = "pipeline1a", starting_file_names = [tempdir + ss for ss in ("a.1", "b.1")])
+ pipeline1b = make_pipeline1(pipeline_name = "pipeline1b", starting_file_names = [tempdir + ss for ss in ("c.1", "d.1")])
+
+ # The Third pipeline is a clone of pipeline1b
+ pipeline1c = pipeline1b.clone(new_name = "pipeline1c")
+
+ # Set the "originate" files for pipeline1c to ("e.1" and "f.1")
+ # Otherwise they would use the original ("c.1", "d.1")
+ pipeline1c.set_output(output = [])
+ pipeline1c.set_output(output = [tempdir + ss for ss in ("e.1", "f.1")])
+
+ # Join all pipeline1a-c to pipeline2
+ pipeline2 = make_pipeline2()
+ pipeline2.set_input(input = [pipeline1a, pipeline1b, pipeline1c])
+
+
+ pipeline2.printout_graph("test.svg", "svg", [task_m_to_1])
+ pipeline2.printout(verbose = 0)
+ pipeline2.run(multiprocess = 10, verbose = 0)
+
+
+ class Test_task(unittest.TestCase):
+
+ def tearDown (self):
+ """
+ """
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+
+
+ def test_subpipelines (self):
+
+ run_pipeline()
+
+ # Check that the output reflecting the pipeline topology is correct.
+ correct_output = 'tempdir/a.1.55=tempdir/a.1.44+tempdir/a.1.33+tempdir/a.1.22+tempdir/a.1=; tempdir/testdir/whatever.txt=; ; ' \
+ 'tempdir/b.1.55=tempdir/b.1.44+tempdir/b.1.33+tempdir/b.1.22+tempdir/b.1=; tempdir/testdir/whatever.txt=; ; ' \
+ 'tempdir/c.1.55=tempdir/c.1.44+tempdir/c.1.33+tempdir/c.1.22+tempdir/c.1=; tempdir/testdir/whatever.txt=; ; ' \
+ 'tempdir/d.1.55=tempdir/d.1.44+tempdir/d.1.33+tempdir/d.1.22+tempdir/d.1=; tempdir/testdir/whatever.txt=; ; ' \
+ 'tempdir/e.1.55=tempdir/e.1.44+tempdir/e.1.33+tempdir/e.1.22+tempdir/e.1=; tempdir/testdir/whatever.txt=; ; ' \
+ 'tempdir/f.1.55=tempdir/f.1.44+tempdir/f.1.33+tempdir/f.1.22+tempdir/f.1=; tempdir/testdir/whatever.txt=; ; '
+ with open(tempdir + "final.output") as real_output:
+ real_output_str = real_output.read()
+ self.assertEqual(correct_output, real_output_str)
+
+
+
+ if __name__ == '__main__':
+ unittest.main()
+
+ ..
+ python
diff --git a/doc/tutorials/new_tutorial/checkpointing_code.rst b/doc/tutorials/new_tutorial/checkpointing_code.rst
index 27632e8..e7363da 100644
--- a/doc/tutorials/new_tutorial/checkpointing_code.rst
+++ b/doc/tutorials/new_tutorial/checkpointing_code.rst
@@ -14,10 +14,44 @@
************************************************************************
-Code for .:ref:`suffix() <decorators.suffix>` example
+Code for the "Interrupting tasks" example
************************************************************************
+
+ .. <<python
+
.. code-block:: python
+
+
from ruffus import *
+ from ruffus import *
+ import sys, time
+
+ # create initial files
+ @originate(['job1.start'])
+ def create_initial_files(output_file):
+ with open(output_file, "w") as oo: pass
+
+
+ #---------------------------------------------------------------
+ #
+ # long task to interrupt
+ #
+ @transform(create_initial_files, suffix(".start"), ".output")
+ def long_task(input_files, output_file):
+ with open(output_file, "w") as ff:
+ ff.write("Unfinished...")
+ # sleep for 2 seconds here so you can interrupt me
+ sys.stderr.write("Job started. Press ^C to interrupt me now...\n")
+ time.sleep(2)
+ ff.write("\nFinished")
+ sys.stderr.write("Job completed.\n")
+
+
+ # Run
+ pipeline_run([long_task])
+
+ ..
+ python
diff --git a/doc/tutorials/new_tutorial/flowchart_colours_code.rst b/doc/tutorials/new_tutorial/flowchart_colours_code.rst
index 13bb236..0a46092 100644
--- a/doc/tutorials/new_tutorial/flowchart_colours_code.rst
+++ b/doc/tutorials/new_tutorial/flowchart_colours_code.rst
@@ -223,7 +223,7 @@ Code
#
# Overriding colours
#
- if options.colour_scheme_index == None:
+ if options.colour_scheme_index is None:
custom_flow_chart_colour_scheme["Vicious cycle"]["linecolor"] = '"#FF3232"'
custom_flow_chart_colour_scheme["Pipeline"]["fontcolor"] = '"#FF3232"'
custom_flow_chart_colour_scheme["Key"]["fontcolor"] = "black"
diff --git a/doc/tutorials/new_tutorial/parallel.rst b/doc/tutorials/new_tutorial/parallel.rst
index 90a825f..a77c2da 100644
--- a/doc/tutorials/new_tutorial/parallel.rst
+++ b/doc/tutorials/new_tutorial/parallel.rst
@@ -17,6 +17,7 @@
* :ref:`@parallel<decorators.parallel>` syntax in detail
+.. _new_manual.parallel:
***************************************
**@parallel**
diff --git a/doc/tutorials/new_tutorial/pipeline_printout_graph.rst b/doc/tutorials/new_tutorial/pipeline_printout_graph.rst
index 5076efe..ce3afac 100644
--- a/doc/tutorials/new_tutorial/pipeline_printout_graph.rst
+++ b/doc/tutorials/new_tutorial/pipeline_printout_graph.rst
@@ -110,6 +110,9 @@ Circular dependency errors in pipelines!
.. image:: ../../images/simple_tutorial_complex_flowchart_error.png
:scale: 70
+
+.. _new_manual.graphviz:
+
==========================================================================================
``@graphviz``: Customising the appearance of each task
==========================================================================================
diff --git a/ruffus/__init__.py b/ruffus/__init__.py
index 301ee73..b6349ba 100644
--- a/ruffus/__init__.py
+++ b/ruffus/__init__.py
@@ -24,9 +24,26 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#################################################################################
-#from graph import *
-#from print_dependencies import *
-from .task import pipeline_printout, pipeline_printout_graph, pipeline_run, pipeline_get_task_names, register_cleanup, check_if_uptodate, active_if, split, transform, merge, collate, files, files_re, follows, parallel, stderr_logger, black_hole_logger, suffix, regex, inputs, add_inputs, touch_file, combine, mkdir, output_from, posttask, JobSignalledBreak, runtime_parameter, jobs_limit, formatter, subdivide, originate, graphviz
+# pipeline functions
+from .task import Pipeline, Task
+
+# pipeline functions
+from .task import pipeline_printout, pipeline_printout_graph, pipeline_run, pipeline_get_task_names, register_cleanup
+
+# decorators
+from .task import originate, split, subdivide, transform, merge, collate, follows
+
+# esoteric decorators
+from .task import files, parallel
+
+# filter / indicators
+from .task import suffix, regex, formatter, inputs, add_inputs, touch_file
+
+# deprecated
+from .task import files_re, combine
+
+from .task import check_if_uptodate, active_if, jobs_limit, graphviz, mkdir, output_from, posttask
+from .task import stderr_logger, black_hole_logger, JobSignalledBreak, runtime_parameter
from .graph import graph_colour_demo_printout
from .file_name_parameters import needs_update_check_modify_time
from . import cmdline
diff --git a/ruffus/cmdline.py b/ruffus/cmdline.py
index 5b8937b..c909a09 100644
--- a/ruffus/cmdline.py
+++ b/ruffus/cmdline.py
@@ -675,7 +675,7 @@ def handle_verbose (options):
#
# verbosity specified manually or deliberately disabled: use that
#
- if options.verbose == None or isinstance(options.verbose, int):
+ if options.verbose is None or isinstance(options.verbose, int):
# verbose_abbreviated_path default to None unless set explicity by the user
# in which case we shall prudently not override it!
# verbose_abbreviated_path of None will be set to the default at
@@ -779,7 +779,7 @@ def run (options, **extra_options):
elif options.flowchart:
appropriate_options = get_extra_options_appropriate_for_command (extra_pipeline_printout_graph_options, extra_options)
- task.pipeline_printout_graph ( open(options.flowchart, "w"),
+ task.pipeline_printout_graph ( open(options.flowchart, "wb"),
options.flowchart_format,
options.target_tasks,
options.forced_tasks,
@@ -810,7 +810,7 @@ def run (options, **extra_options):
extra_options["logger"] = None
if extra_options["logger"] == False:
extra_options["logger"] = task.black_hole_logger
- elif extra_options["logger"] == None:
+ elif extra_options["logger"] is None:
extra_options["logger"] = task.stderr_logger
appropriate_options = get_extra_options_appropriate_for_command (extra_pipeline_run_options, extra_options)
task.pipeline_run( options.target_tasks,
diff --git a/ruffus/drmaa_wrapper.py b/ruffus/drmaa_wrapper.py
index 007fd34..1e8d261 100644
--- a/ruffus/drmaa_wrapper.py
+++ b/ruffus/drmaa_wrapper.py
@@ -78,7 +78,7 @@ class error_drmaa_job(Exception):
#_________________________________________________________________________________________
def read_stdout_stderr_from_files( stdout_path, stderr_path, logger = None, cmd_str = "", tries=5):
- '''
+ """
Reads the contents of two specified paths and returns the strings
Thanks to paranoia approach contributed by Andreas Heger:
@@ -91,7 +91,7 @@ def read_stdout_stderr_from_files( stdout_path, stderr_path, logger = None, cmd_
Returns tuple of stdout and stderr.
- '''
+ """
#
# delay up to 10 seconds until files are ready
#
@@ -215,7 +215,7 @@ def write_job_script_to_temp_file( cmd_str, job_script_directory, job_name, job_
# run_job_using_drmaa
#_________________________________________________________________________________________
-def run_job_using_drmaa (cmd_str, job_name = None, job_other_options = "", job_script_directory = None, job_environment = None, working_directory = None, retain_job_scripts = False, logger = None, drmaa_session = None, verbose = False):
+def run_job_using_drmaa (cmd_str, job_name = None, job_other_options = "", job_script_directory = None, job_environment = None, working_directory = None, retain_job_scripts = False, logger = None, drmaa_session = None, verbose = 0):
"""
Runs specified command remotely using drmaa,
@@ -226,7 +226,7 @@ def run_job_using_drmaa (cmd_str, job_name = None, job_other_options = "", job_s
#
# used specified session else module session
#
- if drmaa_session == None:
+ if drmaa_session is None:
raise error_drmaa_job( "Please specify a drmaa_session in run_job()")
#
@@ -261,11 +261,11 @@ def run_job_using_drmaa (cmd_str, job_name = None, job_other_options = "", job_s
msg = str(exceptionValue)
# ignore message 24 in PBS
# code 24: drmaa: Job finished but resource usage information and/or termination status could not be provided.":
- if not msg.message.startswith("code 24"): raise
+ if not msg.startswith("code 24"): raise
if logger:
- logger.log(MESSAGE, "Warning %s\n"
- "The original command was:\n%s\njobid=jobid\n"
- (msg.message, cmd_str,jobid) )
+ logger.info("Warning %s\n"
+ "The original command was:\n%s\njobid=jobid\n"
+ (msg.message, cmd_str,jobid) )
job_info = None
@@ -281,20 +281,33 @@ def run_job_using_drmaa (cmd_str, job_name = None, job_other_options = "", job_s
(cmd_str,
jobid,
job_script_path))
- if stderr:
- job_info_str += "The stderr was: \n%s\n\n" % ("".join( stderr))
- if stdout:
- job_info_str += "The stdout was: \n%s\n\n" % ("".join( stdout))
+
+ def stderr_stdout_to_str (stderr, stdout):
+ """
+ Concatenate stdout and stderr to string
+ """
+ result = ""
+ if stderr:
+ result += "The stderr was: \n%s\n\n" % ("".join( stderr))
+ if stdout:
+ result += "The stdout was: \n%s\n\n" % ("".join( stdout))
+ return result
#
# Throw if failed
#
if job_info:
job_info_str += "Resources used: %s " % (job_info.resourceUsage)
- if job_info.hasExited:
+ if job_info.wasAborted:
+ raise error_drmaa_job( "The drmaa command was never ran but used %s:\n%s"
+ % (job_info.exitStatus, job_info_str + stderr_stdout_to_str (stderr, stdout)))
+ elif job_info.hasSignal:
+ raise error_drmaa_job( "The drmaa command was terminated by signal %i:\n%s"
+ % (job_info.exitStatus, job_info_str + stderr_stdout_to_str (stderr, stdout)))
+ elif job_info.hasExited:
if job_info.exitStatus:
raise error_drmaa_job( "The drmaa command was terminated by signal %i:\n%s"
- % (job_info.exitStatus, job_info_str))
+ % (job_info.exitStatus, job_info_str + stderr_stdout_to_str (stderr, stdout)))
#
# Decorate normal exit with some resource usage information
#
@@ -326,12 +339,6 @@ def run_job_using_drmaa (cmd_str, job_name = None, job_other_options = "", job_s
logger.info("Drmaa command successfully ran %s" % cmd_str)
except:
logger.info("Drmaa command used %s in running %s" % (job_info.resourceUsage, cmd_str))
- elif job_info.wasAborted:
- raise error_drmaa_job( "The drmaa command was never ran but used %s:\n%s"
- % (job_info.resourceUsage, job_info_str))
- elif job_info.hasSignal:
- raise error_drmaa_job( "The drmaa command was terminated by signal %i:\n%s"
- % (job_info.terminatingSignal, job_info_str))
#
# clean up job template
@@ -446,7 +453,7 @@ def touch_output_files (cmd_str, output_files, logger = None):
def run_job(cmd_str, job_name = None, job_other_options = None, job_script_directory = None,
job_environment = None, working_directory = None, logger = None,
drmaa_session = None, retain_job_scripts = False,
- run_locally = False, output_files = None, touch_only = False, verbose = False):
+ run_locally = False, output_files = None, touch_only = False, verbose = 0):
"""
Runs specified command either using drmaa, or locally or only in simulation (touch the output files only)
"""
@@ -458,5 +465,4 @@ def run_job(cmd_str, job_name = None, job_other_options = None, job_script_direc
if run_locally:
return run_job_locally (cmd_str, logger)
-
return run_job_using_drmaa (cmd_str, job_name, job_other_options, job_script_directory, job_environment, working_directory, retain_job_scripts, logger, drmaa_session, verbose)
diff --git a/ruffus/file_name_parameters.py b/ruffus/file_name_parameters.py
index 6902997..b99e8c5 100644
--- a/ruffus/file_name_parameters.py
+++ b/ruffus/file_name_parameters.py
@@ -78,8 +78,6 @@ from .ruffus_utility import *
from . import dbdict
-class t_extra_inputs:
- (ADD_TO_INPUTS, REPLACE_INPUTS, KEEP_INPUTS) = list(range(3))
class t_combinatorics_type:
( COMBINATORICS_PRODUCT, COMBINATORICS_PERMUTATIONS,
@@ -171,15 +169,21 @@ class t_suffix_file_names_transform(t_file_names_transform):
Does the work for generating output / "extra input" / "extra" filenames
replacing a specified suffix
"""
- def __init__ (self, enclosing_task, suffix_object, error_type, descriptor_string):
+ def __init__ (self, enclosing_task, suffix_object, error_type, descriptor_string, output_dir):
self.matching_regex = compile_suffix(enclosing_task, suffix_object, error_type, descriptor_string)
self.matching_regex_str = suffix_object.args[0]
+ self.output_dir = output_dir
def substitute (self, starting_file_names, pattern):
return regex_replace(starting_file_names[0], self.matching_regex_str, self.matching_regex, pattern)
def substitute_output_files (self, starting_file_names, pattern):
- return regex_replace(starting_file_names[0], self.matching_regex_str, self.matching_regex, pattern, SUFFIX_SUBSTITUTE)
+ res = regex_replace(starting_file_names[0], self.matching_regex_str, self.matching_regex, pattern, SUFFIX_SUBSTITUTE)
+ if self.output_dir == []:
+ return res
+ # N.B. Does not do output directory substitution for extra parameters
+ else:
+ return os.path.join(self.output_dir, os.path.split(res)[1])
class t_regex_file_names_transform(t_file_names_transform):
@@ -340,7 +344,7 @@ def needs_update_check_directory_missing (*params, **kwargs):
raise Exception("Wrong number of arguments in mkdir check %s" % (params,))
missing_directories = []
- for d in get_strings_in_nested_sequence(dirs):
+ for d in get_strings_in_flattened_sequence(dirs):
#print >>sys.stderr, "check directory missing %d " % os.path.exists(d) # DEBUG
if not os.path.exists(d):
missing_directories.append(d)
@@ -351,9 +355,9 @@ def needs_update_check_directory_missing (*params, **kwargs):
if len(missing_directories):
if len(missing_directories) > 1:
- return True, "Directories [%s] are missing" % (", ".join(missing_directories))
+ return True, ": Directories %r are missing" % (", ".join(missing_directories))
else:
- return True, "Directories [%s] is missing" % (missing_directories[0])
+ return True, ": Directories %r is missing" % (missing_directories[0])
return False, "All directories exist"
#_________________________________________________________________________________________
@@ -371,7 +375,7 @@ def check_input_files_exist (*params):
"""
if len(params):
input_files = params[0]
- for f in get_strings_in_nested_sequence(input_files):
+ for f in get_strings_in_flattened_sequence(input_files):
if not os.path.exists(f):
if os.path.lexists(f):
raise MissingInputFileError("No way to run job: "+
@@ -407,8 +411,8 @@ def needs_update_check_exist (*params, **kwargs):
i, o = params[0:2]
- i = get_strings_in_nested_sequence(i)
- o = get_strings_in_nested_sequence(o)
+ i = get_strings_in_flattened_sequence(i)
+ o = get_strings_in_flattened_sequence(o)
#
# build: missing output file
@@ -423,9 +427,9 @@ def needs_update_check_exist (*params, **kwargs):
if not os.path.exists(p):
missing_files.append(p)
if len(missing_files):
- return True, "Missing file%s\n%s" % ("s" if len(missing_files) > 1 else "",
+ return True, "...\n Missing file%s %s" % ("s" if len(missing_files) > 1 else "",
shorten_filenames_encoder (missing_files,
- verbose_abbreviated_path))
+ verbose_abbreviated_path))
#
# missing input -> build only if output absent
@@ -491,8 +495,8 @@ def needs_update_check_modify_time (*params, **kwargs):
return True, ""
i, o = params[0:2]
- i = get_strings_in_nested_sequence(i)
- o = get_strings_in_nested_sequence(o)
+ i = get_strings_in_flattened_sequence(i)
+ o = get_strings_in_flattened_sequence(o)
#
# build: missing output file
@@ -507,24 +511,31 @@ def needs_update_check_modify_time (*params, **kwargs):
if not os.path.exists(p):
missing_files.append(p)
if len(missing_files):
- return True, "Missing file%s\n %s" % ("s" if len(missing_files) > 1 else "",
+ return True, "...\n Missing file%s %s" % ("s" if len(missing_files) > 1 else "",
shorten_filenames_encoder (missing_files,
verbose_abbreviated_path))
+ #
+ # N.B. Checkpointing uses relative paths
+ #
# existing files, but from previous interrupted runs
if task.checksum_level >= CHECKSUM_HISTORY_TIMESTAMPS:
incomplete_files = []
+ set_incomplete_files = set()
func_changed_files = []
+ set_func_changed_files = set()
param_changed_files = []
+ set_param_changed_files = set()
#for io in (i, o):
# for p in io:
# if p not in job_history:
# incomplete_files.append(p)
for p in o:
- if os.path.relpath(p) not in job_history:
+ if os.path.relpath(p) not in job_history and p not in set_incomplete_files:
incomplete_files.append(p)
+ set_incomplete_files.add(p)
if len(incomplete_files):
- return True, "Previous incomplete run leftover%s:\n %s" % ("s" if len(incomplete_files) > 1 else "",
+ return True, "Uncheckpointed file%s (left over from a failed run?):\n %s" % ("s" if len(incomplete_files) > 1 else "",
shorten_filenames_encoder (incomplete_files,
verbose_abbreviated_path))
# check if function that generated our output file has changed
@@ -533,11 +544,15 @@ def needs_update_check_modify_time (*params, **kwargs):
old_chksum = job_history[rel_o_f_n]
new_chksum = JobHistoryChecksum(rel_o_f_n, None, params[2:], task)
if task.checksum_level >= CHECKSUM_FUNCTIONS_AND_PARAMS and \
- new_chksum.chksum_params != old_chksum.chksum_params:
- param_changed_files.append(rel_o_f_n)
+ new_chksum.chksum_params != old_chksum.chksum_params and \
+ o_f_n not in set_func_changed_files:
+ param_changed_files.append(o_f_n)
+ set_param_changed_files.add(o_f_n)
elif task.checksum_level >= CHECKSUM_FUNCTIONS and \
- new_chksum.chksum_func != old_chksum.chksum_func:
- func_changed_files.append(rel_o_f_n)
+ new_chksum.chksum_func != old_chksum.chksum_func and \
+ o_f_n not in set_func_changed_files:
+ func_changed_files.append(o_f_n)
+ set_func_changed_files.add(o_f_n)
if len(func_changed_files):
return True, "Pipeline function has changed:\n %s" % (shorten_filenames_encoder (func_changed_files,
@@ -627,8 +642,15 @@ def needs_update_check_modify_time (*params, **kwargs):
# for output files, we need to check modification time *in addition* to
# function and argument checksums...
for output_file_name in o:
- rel_output_file_name = os.path.relpath(output_file_name)
+ #
+ # Ignore output files which are just symbolic links to input files or passed through
+ # from input to output
+ #
real_file_name = os.path.realpath(output_file_name)
+ if real_file_name in real_input_file_names:
+ continue
+
+ rel_output_file_name = os.path.relpath(output_file_name)
file_timestamp = os.path.getmtime(output_file_name)
if task.checksum_level >= CHECKSUM_HISTORY_TIMESTAMPS:
old_chksum = job_history[rel_output_file_name]
@@ -640,8 +662,7 @@ def needs_update_check_modify_time (*params, **kwargs):
mtime = old_chksum.mtime
else:
mtime = file_timestamp
- if real_file_name not in real_input_file_names:
- file_times[1].append(mtime)
+ file_times[1].append(mtime)
filename_to_times[1].append((mtime, output_file_name))
@@ -696,19 +717,16 @@ def file_names_from_tasks_globs(files_task_globs,
Replaces glob specifications and tasks with actual files / task output
"""
- #
- # N.B. get_output_files() should never have the flattened flag == True
- # do that later in get_strings_in_nested_sequence
- #
# special handling for chaining tasks which conceptual have a single job
# i.e. @merge and @files/@parallel with single job parameters
- if files_task_globs.params.__class__.__name__ == '_task' and do_not_expand_single_job_tasks:
- return files_task_globs.params.get_output_files(True, runtime_data)
+ if files_task_globs.params.__class__.__name__ == 'Task' and do_not_expand_single_job_tasks:
+ return files_task_globs.params._get_output_files(True, runtime_data)
task_or_glob_to_files = dict()
+
# look up globs and tasks
for g in files_task_globs.globs:
# check whether still is glob pattern after transform
@@ -716,7 +734,7 @@ def file_names_from_tasks_globs(files_task_globs,
if is_glob(g):
task_or_glob_to_files[g] = sorted(glob.glob(g))
for t in files_task_globs.tasks:
- of = t.get_output_files(False, runtime_data)
+ of = t._get_output_files(False, runtime_data)
task_or_glob_to_files[t] = of
for n in files_task_globs.runtime_data_names:
data_name = n.args[0]
@@ -727,8 +745,6 @@ def file_names_from_tasks_globs(files_task_globs,
"the runtime parameter " +
"'%s' which is missing " % data_name)
-
-
return expand_nested_tasks_or_globs(files_task_globs.params, task_or_glob_to_files)
@@ -843,7 +859,7 @@ def args_param_factory (orig_args):
# ]
#
#_________________________________________________________________________________________
-def files_param_factory (input_files_task_globs, flatten_input,
+def files_param_factory (input_files_task_globs,
do_not_expand_single_job_tasks, output_extras):
"""
Factory for functions which
@@ -869,10 +885,7 @@ def files_param_factory (input_files_task_globs, flatten_input,
for input_spec, output_extra_param in zip(input_files_task_globs.param_iter(), output_extras):
input_param = file_names_from_tasks_globs(input_spec, runtime_data, do_not_expand_single_job_tasks)
- if flatten_input:
- yield_param = (get_strings_in_nested_sequence(input_param),) + output_extra_param
- else:
- yield_param = (input_param, ) + output_extra_param
+ yield_param = (input_param, ) + output_extra_param
yield yield_param, yield_param
return iterator
@@ -897,7 +910,6 @@ def split_param_factory (input_files_task_globs, output_files_task_globs, *extra
Factory for task_split
"""
def iterator(runtime_data):
- # flattened = False
# do_not_expand_single_job_tasks = True
#
@@ -917,6 +929,56 @@ def split_param_factory (input_files_task_globs, output_files_task_globs, *extra
#_________________________________________________________________________________________
+# merge_param_factory
+
+#_________________________________________________________________________________________
+def merge_param_factory (input_files_task_globs,
+ output_param,
+ *extra_params):
+ """
+ Factory for task_merge
+ """
+ #
+ def iterator(runtime_data):
+ # do_not_expand_single_job_tasks = True
+ input_param = file_names_from_tasks_globs(input_files_task_globs, runtime_data, True)
+ yield_param = (input_param, output_param) + extra_params
+ yield yield_param, yield_param
+
+ return iterator
+
+
+#_________________________________________________________________________________________
+
+# originate_param_factory
+
+#_________________________________________________________________________________________
+def originate_param_factory (list_output_files_task_globs,
+ *extra_params):
+ """
+ Factory for task_originate
+ """
+ #
+ def iterator(runtime_data):
+ for output_files_task_globs in list_output_files_task_globs:
+ output_param = file_names_from_tasks_globs(output_files_task_globs, runtime_data)
+ output_param_logging = file_names_from_tasks_globs(output_files_task_globs.unexpanded_globs(), runtime_data)
+ yield (None, output_param) + tuple(extra_params), (None, output_param_logging) + tuple(extra_params)
+
+ return iterator
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# param_factories
+
+# ... which take inputs(), add_inputs(), suffix(), regex(), formatter()
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+
+#_________________________________________________________________________________________
+
# input_param_to_file_name_list
#_________________________________________________________________________________________
@@ -931,7 +993,7 @@ def input_param_to_file_name_list (input_params):
==> generator of flat list of strings (file_names)
"""
for per_job_input_param in input_params:
- flattened_list_of_file_names = get_strings_in_nested_sequence(per_job_input_param)
+ flattened_list_of_file_names = get_strings_in_flattened_sequence(per_job_input_param)
yield per_job_input_param, flattened_list_of_file_names
@@ -949,7 +1011,7 @@ def list_input_param_to_file_name_list (input_params):
==> generator of flat list of strings (file_names)
"""
for per_job_input_param_list in input_params:
- list_of_flattened_list_of_file_names = [ get_strings_in_nested_sequence(ii) for ii in per_job_input_param_list]
+ list_of_flattened_list_of_file_names = [ get_strings_in_flattened_sequence(ii) for ii in per_job_input_param_list]
yield per_job_input_param_list, list_of_flattened_list_of_file_names
@@ -971,7 +1033,9 @@ def yield_io_params_per_job (input_params,
Helper function for
transform_param_factory and
collate_param_factory and
- subdivide_param_factory
+ subdivide_param_factory and
+ combinatorics_param_factory and
+ product_param_factory
*********************************************************
@@ -1036,8 +1100,8 @@ def yield_io_params_per_job (input_params,
# just like output params
#
# So we do (2) first, ignoring tasks, then (1)
- if extra_input_files_task_globs != None:
-
+ if extra_input_files_task_globs:
+ # DEBUGGG
extra_inputs = extra_input_files_task_globs.file_names_transformed (filenames, file_names_transform)
#
@@ -1047,6 +1111,8 @@ def yield_io_params_per_job (input_params,
input_param = file_names_from_tasks_globs(extra_inputs, runtime_data)
elif replace_inputs == t_extra_inputs.ADD_TO_INPUTS:
input_param = (orig_input_param,) + file_names_from_tasks_globs(extra_inputs, runtime_data)
+ else:
+ input_param = orig_input_param
else:
input_param = orig_input_param
@@ -1062,10 +1128,10 @@ def yield_io_params_per_job (input_params,
# before glob matching
#
output_pattern_transformed = output_pattern.file_names_transformed (filenames, file_names_transform)
- output_param = file_names_from_tasks_globs(output_pattern_transformed, runtime_data)
- output_param_logging= file_names_from_tasks_globs(output_pattern_transformed.unexpanded_globs(), runtime_data)
- yield ( (input_param, output_param ) + extra_params,
- (input_param, output_param_logging ) + extra_params)
+ output_param = file_names_from_tasks_globs(output_pattern_transformed, runtime_data)
+ output_param_unglobbed= file_names_from_tasks_globs(output_pattern_transformed.unexpanded_globs(), runtime_data)
+ yield ( (input_param, output_param ) + extra_params,
+ (input_param, output_param_unglobbed ) + extra_params)
else:
# output
@@ -1106,7 +1172,6 @@ def yield_io_params_per_job (input_params,
#_________________________________________________________________________________________
def subdivide_param_factory (input_files_task_globs,
- flatten_input,
file_names_transform,
extra_input_files_task_globs,
replace_inputs,
@@ -1123,8 +1188,6 @@ def subdivide_param_factory (input_files_task_globs,
#
input_params = file_names_from_tasks_globs(input_files_task_globs, runtime_data)
- if flatten_input:
- input_params = get_strings_in_nested_sequence(input_params)
if not len(input_params):
return []
@@ -1150,7 +1213,6 @@ def subdivide_param_factory (input_files_task_globs,
#_________________________________________________________________________________________
def combinatorics_param_factory(input_files_task_globs,
- flatten_input,
combinatorics_type,
k_tuple,
file_names_transform,
@@ -1172,8 +1234,6 @@ def combinatorics_param_factory(input_files_task_globs,
if not len(input_params):
return
- if flatten_input:
- input_params = get_strings_in_nested_sequence(input_params)
if combinatorics_type == t_combinatorics_type.COMBINATORICS_PERMUTATIONS:
combinatoric_iter = itertools.permutations(input_params, k_tuple)
@@ -1203,7 +1263,6 @@ def combinatorics_param_factory(input_files_task_globs,
#_________________________________________________________________________________________
def product_param_factory ( list_input_files_task_globs,
- flatten_input,
file_names_transform,
extra_input_files_task_globs,
replace_inputs,
@@ -1230,8 +1289,6 @@ def product_param_factory ( list_input_files_task_globs,
if not len(input_params):
return
- if flatten_input:
- input_params_list = [get_strings_in_nested_sequence(ii) for ii in input_params_list]
for y in yield_io_params_per_job (list_input_param_to_file_name_list(itertools.product(*input_params_list)),
file_names_transform,
@@ -1253,7 +1310,6 @@ def product_param_factory ( list_input_files_task_globs,
#_________________________________________________________________________________________
def transform_param_factory (input_files_task_globs,
- flatten_input,
file_names_transform,
extra_input_files_task_globs,
replace_inputs,
@@ -1270,8 +1326,6 @@ def transform_param_factory (input_files_task_globs,
#
input_params = file_names_from_tasks_globs(input_files_task_globs, runtime_data)
- if flatten_input:
- input_params = get_strings_in_nested_sequence(input_params)
if not len(input_params):
return
@@ -1296,7 +1350,6 @@ def transform_param_factory (input_files_task_globs,
#_________________________________________________________________________________________
def collate_param_factory (input_files_task_globs,
- flatten_input,
file_names_transform,
extra_input_files_task_globs,
replace_inputs,
@@ -1316,8 +1369,6 @@ def collate_param_factory (input_files_task_globs,
#
input_params = file_names_from_tasks_globs(input_files_task_globs, runtime_data)
- if flatten_input:
- input_params = get_strings_in_nested_sequence(input_params)
if not len(input_params):
return
@@ -1356,44 +1407,3 @@ def collate_param_factory (input_files_task_globs,
return iterator
-
-#_________________________________________________________________________________________
-
-# merge_param_factory
-
-#_________________________________________________________________________________________
-def merge_param_factory (input_files_task_globs,
- output_param,
- *extra_params):
- """
- Factory for task_merge
- """
- #
- def iterator(runtime_data):
- # flattened = False
- # do_not_expand_single_job_tasks = True
- input_param = file_names_from_tasks_globs(input_files_task_globs, runtime_data, True)
- yield_param = (input_param, output_param) + extra_params
- yield yield_param, yield_param
-
- return iterator
-
-
-#_________________________________________________________________________________________
-
-# originate_param_factory
-
-#_________________________________________________________________________________________
-def originate_param_factory (list_output_files_task_globs, extras):
- """
- Factory for task_originate
- """
- #
- def iterator(runtime_data):
- for output_files_task_globs in list_output_files_task_globs:
- output_param = file_names_from_tasks_globs(output_files_task_globs, runtime_data)
- output_param_logging = file_names_from_tasks_globs(output_files_task_globs.unexpanded_globs(), runtime_data)
- yield (None, output_param) + tuple(extras), (None, output_param_logging) + tuple(extras)
-
- return iterator
-
diff --git a/ruffus/graph.py b/ruffus/graph.py
index 1f61e63..d42d91d 100644
--- a/ruffus/graph.py
+++ b/ruffus/graph.py
@@ -70,45 +70,50 @@ class node (object):
_all_nodes = list()
_name_to_node = dict()
+ _index_to_node = dict()
_global_node_index = 0
- one_to_one = 0
- many_to_many = 1
- one_to_many = 2
- many_to_one = 3
+ _one_to_one = 0
+ _many_to_many = 1
+ _one_to_many = 2
+ _many_to_one = 3
@staticmethod
- def get_leaves ():
+ def _get_leaves ():
for n in node._all_nodes:
if len(n._inward) == 0:
yield n
@staticmethod
- def get_roots ():
+ def _get_roots ():
for n in node._all_nodes:
if len(n._outward) == 0:
yield n
@staticmethod
- def count_nodes ():
+ def _count_nodes ():
return len(_all_nodes)
@staticmethod
- def dump_tree_as_str ():
+ def _dump_tree_as_str ():
"""
dumps entire tree
"""
- return ("%d nodes " % node.count_nodes()) + "\n" + \
- "\n".join([x.fullstr() for x in node._all_nodes])
+ return ("%d nodes " % node._count_nodes()) + "\n" + \
+ "\n".join([x._fullstr() for x in node._all_nodes])
@staticmethod
- def lookup_node_from_name (name):
+ def _lookup_node_from_name (name):
return node._name_to_node[name]
@staticmethod
- def is_node (name):
+ def _lookup_node_from_index (index):
+ return node._index_to_node[index]
+
+ @staticmethod
+ def _is_node (name):
return name in node._name_to_node
@@ -127,8 +132,8 @@ class node (object):
#
# make sure node name is unique
#
- if name in node._name_to_node:
- raise error_duplicate_node_name("[%s] has already been added" % name)
+ #if name in node._name_to_node:
+ # raise error_duplicate_node_name("[%s] has already been added" % name)
self.__dict__.update(args)
self._inward = list()
@@ -143,19 +148,20 @@ class node (object):
# for looking up node for name
#
node._all_nodes.append(self)
- node._name_to_node[name] = self
+ node._name_to_node[self._name] = self
+ node._index_to_node[self._node_index] = self
#_____________________________________________________________________________________
- # add_child
+ # _add_child
#_____________________________________________________________________________________
- def add_child(self, child, no_duplicates = True):
+ def _add_child(self, child):
"""
connect edges
"""
# do not add duplicates
- if no_duplicates and child in self._outward:
+ if child in self._outward:
return child
self._outward.append(child)
@@ -164,17 +170,84 @@ class node (object):
#_____________________________________________________________________________________
- # inward/outward
+ # _remove_child
+
+ #_____________________________________________________________________________________
+ def _remove_child(self, child):
+ """
+ disconnect edges
+ """
+
+ if child in self._outward:
+ self._outward.remove(child)
+ if self in child._inward:
+ child._inward.remove(self)
+ return child
+ #_____________________________________________________________________________________
+
+ # _add_parent
+
+ #_____________________________________________________________________________________
+ def _add_parent(self, parent):
+ """
+ connect edges
+ """
+ # do not add duplicates
+ if parent in self._inward:
+ return parent
+
+ self._inward.append(parent)
+ parent._outward.append(self)
+ return parent
+
+ #_____________________________________________________________________________________
+
+ # _remove_all_parents
+
+ #_____________________________________________________________________________________
+ def _remove_all_parents(self):
+ """
+ disconnect edges
+ """
+
+ # remove self from parent
+ for parent in self._inward:
+ if self in parent._outward:
+ parent._outward.remove(self)
+
+ # clear self
+ self._inward = []
+
+ return self
+
+ #_____________________________________________________________________________________
+
+ # _remove_parent
#_____________________________________________________________________________________
- def outward (self):
+ def _remove_parent(self, parent):
+ """
+ disconnect edges
+ """
+
+ if parent in self._inward:
+ self._inward.remove(parent)
+ if self in parent._outward:
+ parent._outward.remove(self)
+ return parent
+ #_____________________________________________________________________________________
+
+ # _get_inward/_get_outward
+
+ #_____________________________________________________________________________________
+ def _get_outward (self):
"""
just in case we need to return inward when we mean outward!
(for reversed graphs)
"""
return self._outward
- def inward (self):
+ def _get_inward (self):
"""
just in case we need to return inward when we mean outward!
(for reversed graphs)
@@ -184,10 +257,10 @@ class node (object):
#_____________________________________________________________________________________
- # fullstr
+ # _fullstr
#_____________________________________________________________________________________
- def fullstr(self):
+ def _fullstr(self):
"""
Full dump. Normally edges are not printed out
Everything is indented except name
@@ -209,9 +282,8 @@ class node (object):
#_____________________________________________________________________________________
def __str__ (self):
"""
- Dump.
Print everything except lists of edges
- Everything is indented except name
+ Useful for debugging
"""
self_desc = list()
for k,v in sorted(self.__dict__.items(), reverse=True):
@@ -220,16 +292,16 @@ class node (object):
continue
else:
self_desc.append(indent + str(k) + "=" + str(v))
- return "\n".join(self_desc)
+ return " Task = " + "\n".join(self_desc)
#_____________________________________________________________________________________
- # signal
+ # _signalled
#
#_____________________________________________________________________________________
- def signal (self, extra_data_for_signal = None):
+ def _signalled (self, extra_data_for_signal = None):
"""
Signals whether depth first search ends without this node
"""
@@ -255,9 +327,9 @@ class node_to_json(json.JSONEncoder):
if isinstance(obj, node):
return obj._name, {
"index": obj._node_index,
- "signal": obj._signal,
- "inward": [n._name for n in obj._inward],
- "outward": [n._name for n in obj._outward],
+ "_signal": obj._signal,
+ "_get_inward": [n._name for n in obj._inward],
+ "_get_outward": [n._name for n in obj._outward],
}
return json.JSONEncoder.default(self, obj)
@@ -275,6 +347,12 @@ class node_to_json(json.JSONEncoder):
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+def default_signalled (node, extra_data):
+ """
+ Depth first search stops when node._signalled return True
+ """
+ return node._signalled(extra_data)
+
class topological_sort_visitor (object):
"""
topological sort
@@ -294,7 +372,8 @@ class topological_sort_visitor (object):
#_____________________________________________________________________________________
def __init__ (self, forced_dfs_nodes,
node_termination = END_ON_SIGNAL,
- extra_data_for_signal = None):
+ extra_data_for_signal = None,
+ signal_callback = None):
"""
list of saved results
"""
@@ -306,6 +385,8 @@ class topological_sort_visitor (object):
self._back_nodes = set()
self._signalling_nodes = set()
+ self.signal_callback = signal_callback
+
# keep order for tree traversal later
self._examined_edges = list()
# keep order for topological sorted results
@@ -446,10 +527,10 @@ class topological_sort_visitor (object):
#
backtracked_nodes = set()
for f,t,n in self._examined_edges[cycle_start:cycle_end]:
- if t == None:
+ if t is None:
backtracked_nodes.add(n)
for f,t,n in self._examined_edges[cycle_start:cycle_end]:
- if f == None or f in backtracked_nodes or t in backtracked_nodes:
+ if f is None or f in backtracked_nodes or t in backtracked_nodes:
continue
cycle_edges.add((f,t))
self._back_nodes.add(f)
@@ -516,7 +597,7 @@ class topological_sort_visitor (object):
# Note that _forced_dfs_nodes is ignored
#
if self._node_termination == self.NOTE_NODE_SIGNAL:
- if node.signal(self._extra_data_for_signal):
+ if self.signal_callback(node, self._extra_data_for_signal):
self._signalling_nodes.add(node)
return False
@@ -527,14 +608,14 @@ class topological_sort_visitor (object):
if node in self._forced_dfs_nodes:
## Commented out code lets us save self_terminating_nodes even when
## they have been overridden by _forced_dfs_nodes
- #if node.signal():
+ #if self.signal_callback(node, self._extra_data_for_signal):
# self._signalling_nodes.add(node)
return False
#
# OK. Go by what the node wants then
#
- if node.signal(self._extra_data_for_signal):
+ if self.signal_callback(node, self._extra_data_for_signal):
self._signalling_nodes.add(node)
return True
return False
@@ -714,7 +795,7 @@ def depth_first_visit(u, visitor, colours, outedges_func):
visitor.finish_vertex(u)
-def depth_first_search(starting_nodes, visitor, outedges_func = node.outward):
+def depth_first_search(starting_nodes, visitor, outedges_func = node._get_inward):
"""
depth_first_search
go through all starting points and DFV on each of them
@@ -750,13 +831,13 @@ def topologically_sorted_nodes( to_leaves,
gather_all_non_signalled = True,
test_all_signals = False,
extra_data_for_signal = None,
- checksum_level = None):
+ signal_callback = None):
"""
Get all nodes which are children of to_leaves
in topological sorted order
- Defaults to including all nodes which are non-signalled
- i.e. includes the *last* non-signalling node on each branch
+ Defaults to including all nodes which are non-signalled and their dependents (via include_any_children())
+ i.e. includes the *last* non-signalling node on each branch and all the way up the tree
Otherwise stops at each branch just before signalling node
@@ -771,7 +852,7 @@ def topologically_sorted_nodes( to_leaves,
force_start_from = True to get the whole tree irrespective of signalling
- Rewritten to minimise calls to node.signal()
+ Rewritten to minimise calls to node._signalled()
"""
@@ -782,8 +863,9 @@ def topologically_sorted_nodes( to_leaves,
if test_all_signals:
v = topological_sort_visitor([],
topological_sort_visitor.NOTE_NODE_SIGNAL,
- extra_data_for_signal)
- depth_first_search(to_leaves, v, node.outward)
+ extra_data_for_signal,
+ signal_callback)
+ depth_first_search(to_leaves, v, node._get_inward)
signalling_nodes = v._signalling_nodes
else:
signalling_nodes = set()
@@ -792,10 +874,8 @@ def topologically_sorted_nodes( to_leaves,
#
# get whole tree, ignoring signalling
#
- v = topological_sort_visitor([],
- topological_sort_visitor.IGNORE_NODE_SIGNAL,
- None)
- depth_first_search(to_leaves, v, node.outward)
+ v = topological_sort_visitor([], topological_sort_visitor.IGNORE_NODE_SIGNAL)
+ depth_first_search(to_leaves, v, node._get_inward)
#
# not dag: no further processing
@@ -807,35 +887,52 @@ def topologically_sorted_nodes( to_leaves,
#
- # return entire tree
+ # if force_start_from == True
+ #
+ # return entire tree
#
if force_start_from == True:
return (v.topological_sorted(), v._signalling_nodes, v.dag_violating_edges(),
v.dag_violating_nodes())
-
#
- # If we include these nodes anyway,
- # why bother to check if they do not signal?
- # Expensive signal checking should be minimised
+ # Set of all nodes we are going to return
+ # We will use v.topological_sorted to return them in the right (sorted) order
#
nodes_to_include = set()
- for n in force_start_from:
- if n in nodes_to_include:
- continue
- nodes_to_include.add(n)
- nodes_to_include.update(get_parent_nodes([n]))
+
+ #
+ # If force start from is a list of nodes,
+ # include these and all of its dependents (via include_any_children)
+ #
+ # We don't need to bother to check if they signal (_signalled)
+ # This saves calling the expensive _signalled
+ #
+ if len(force_start_from):
+ nodes_to_include.update(include_any_children(force_start_from))
+ # This should not be necessary because include_any_children also returns self.
+ #for n in force_start_from:
+ # if n in nodes_to_include:
+ # continue
+ # nodes_to_include.add(n)
+ # nodes_to_include.update(include_any_children([n]))
+ #
+ # Now select all nodes from ancestor -> descendant which do not signal (signal_callback() == false)
+ # and select their descendants (via include_any_children())
+ #
+ # Nodes which signal are added to signalling_nodes
+ #
reversed_nodes = v.topological_sorted()
for n in reversed_nodes:
if n in nodes_to_include:
continue
- if not n.signal(extra_data_for_signal):
- nodes_to_include.add(n)
- nodes_to_include.update(get_parent_nodes([n]))
+ if not signal_callback(n, extra_data_for_signal):
+ #nodes_to_include.add(n)
+ nodes_to_include.update(include_any_children([n]))
else:
signalling_nodes.add(n)
#sys.stderr.write(json.dumps(n, cls=node_to_json, sort_keys=1) + "\n")
@@ -845,6 +942,11 @@ def topologically_sorted_nodes( to_leaves,
signalling_nodes,
[],[])
+
+ #
+ # gather_all_non_signalled = False
+ # stop at first signalled
+ #
else:
if force_start_from == True:
@@ -852,28 +954,29 @@ def topologically_sorted_nodes( to_leaves,
# get whole tree, ignoring signalling
#
v = topological_sort_visitor([],
- topological_sort_visitor.IGNORE_NODE_SIGNAL,
- extra_data_for_signal)
+ topological_sort_visitor.IGNORE_NODE_SIGNAL)
else:
#
# End at each branch without including signalling node
# but ignore signalling for forced_nodes_and_dependencies
#
- # Get all parents of forced nodes if necessary
+ # Get forced nodes and all descendants via include_any_children
+ #
forced_nodes_and_dependencies = []
if len(force_start_from):
- forced_nodes_and_dependencies = get_parent_nodes(force_start_from)
+ forced_nodes_and_dependencies = include_any_children(force_start_from)
v = topological_sort_visitor( forced_nodes_and_dependencies,
topological_sort_visitor.END_ON_SIGNAL,
- extra_data_for_signal)
+ extra_data_for_signal,
+ signal_callback)
#
# Forward graph iteration
#
- depth_first_search(to_leaves, v, node.outward)
+ depth_first_search(to_leaves, v, node._get_inward)
if v.not_dag():
v.identify_dag_violating_nodes_and_edges ()
@@ -885,7 +988,7 @@ def topologically_sorted_nodes( to_leaves,
#
def debug_print_nodes(to_leaves):
v = debug_print_visitor()
- depth_first_search(to_leaves, v, node.outward)
+ depth_first_search(to_leaves, v, node._get_inward)
@@ -944,18 +1047,20 @@ def graph_printout_in_dot_format ( stream,
minimal_key_legend = True,
user_colour_scheme = None,
pipeline_name = "Pipeline:",
- extra_data_for_signal = None):
+ extra_data_for_signal = None,
+ signal_callback = None):
"""
print out pipeline dependencies in dot formatting
"""
- (topological_sorted,
- signalling_nodes,
+ (topological_sorted, # tasks_to_run
+ signalling_nodes, # up to date
dag_violating_edges,
dag_violating_nodes) = topologically_sorted_nodes(to_leaves, force_start_from,
gather_all_non_signalled,
test_all_signals,
- extra_data_for_signal)
+ extra_data_for_signal,
+ signal_callback)
#
# N.B. For graph:
@@ -969,8 +1074,8 @@ def graph_printout_in_dot_format ( stream,
#
# print out dependencies in dot format
#
- write_flowchart_in_dot_format(topological_sorted,
- signalling_nodes,
+ write_flowchart_in_dot_format(topological_sorted, # tasks_to_run
+ signalling_nodes, # up to date
dag_violating_edges,
dag_violating_nodes,
stream,
@@ -1004,7 +1109,8 @@ def graph_printout (stream,
pipeline_name = "Pipeline:",
size = (11,8),
dpi = 120,
- extra_data_for_signal = None):
+ extra_data_for_signal = None,
+ signal_callback = None):
"""
print out pipeline dependencies in a variety of formats, using the programme "dot"
an intermediary
@@ -1023,13 +1129,14 @@ def graph_printout (stream,
minimal_key_legend,
user_colour_scheme,
pipeline_name,
- extra_data_for_signal)
+ extra_data_for_signal,
+ signal_callback)
return
# print to dot file
#temp_dot_file = tempfile.NamedTemporaryFile(suffix='.dot', delete=False)
fh, temp_dot_file_name = tempfile.mkstemp(suffix='.dot')
- temp_dot_file = os.fdopen(fh, "w")
+ temp_dot_file = os.fdopen(fh, "wb")
graph_printout_in_dot_format ( temp_dot_file,
to_leaves,
@@ -1043,7 +1150,8 @@ def graph_printout (stream,
minimal_key_legend,
user_colour_scheme,
pipeline_name,
- extra_data_for_signal)
+ extra_data_for_signal,
+ signal_callback)
temp_dot_file.close()
if isinstance(size, tuple):
@@ -1064,7 +1172,7 @@ def graph_printout (stream,
result_str, error_str = proc.communicate()
retcode = proc.returncode
if retcode:
- raise subprocess.CalledProcessError(retcode, cmd + "\n" + "\n".join([result_str, error_str]))
+ raise subprocess.CalledProcessError(retcode, cmd + "\n" + "\n".join([str(result_str), str(error_str)]))
@@ -1083,25 +1191,29 @@ def graph_printout (stream,
if output_format == "svg":
+ # result str is a binary string. I.e. could be .jpg
+ # must turn it into string before we can replace, and then turn it back into binary
+ result_str = result_str.decode()
result_str = result_str.replace("0.12", "0.0px")
+ result_str = result_str.encode()
stream.write(result_str)
#_________________________________________________________________________________________
-# get_parent_nodes
+# include_any_children
#_________________________________________________________________________________________
-def get_parent_nodes (nodes):
+def include_any_children (nodes):
"""
- Get all parent nodes by DFS in the inward direction,
+ Get all children nodes by DFS in the inward direction,
Ignores signals
+ Also includes original nodes in the results
"""
- parent_visitor = topological_sort_visitor([],
- topological_sort_visitor.IGNORE_NODE_SIGNAL)
- depth_first_search(nodes, parent_visitor, node.inward)
- return parent_visitor.topological_sorted()
+ children_visitor = topological_sort_visitor([], topological_sort_visitor.IGNORE_NODE_SIGNAL)
+ depth_first_search(nodes, children_visitor, node._get_outward)
+ return children_visitor.topological_sorted()
#_________________________________________________________________________________________
@@ -1109,23 +1221,24 @@ def get_parent_nodes (nodes):
# get_reachable_nodes
#_________________________________________________________________________________________
-def get_reachable_nodes(nodes, parents_as_well = True):
+def get_reachable_nodes(nodes, children_as_well = True):
"""
Get all nodes which are parents and children of nodes
recursing through the entire tree
+ i.e. go up *and* down tree starting from node
+
1) specify parents_as_well = False
to only get children and not parents of nodes
"""
# look for parents of nodes and start there instead
- if parents_as_well:
- nodes = get_parent_nodes (nodes)
+ if children_as_well:
+ nodes = include_any_children (nodes)
- child_visitor = topological_sort_visitor([],
- topological_sort_visitor.IGNORE_NODE_SIGNAL)
- depth_first_search(nodes, child_visitor, node.outward)
- return child_visitor.topological_sorted()
+ parent_visitor = topological_sort_visitor([], topological_sort_visitor.IGNORE_NODE_SIGNAL)
+ depth_first_search(nodes, parent_visitor, node._get_inward)
+ return parent_visitor.topological_sorted()
#_________________________________________________________________________________________
diff --git a/ruffus/parse_old_style_ruffus.py b/ruffus/parse_old_style_ruffus.py
new file mode 100755
index 0000000..576e2d2
--- /dev/null
+++ b/ruffus/parse_old_style_ruffus.py
@@ -0,0 +1,268 @@
+#!/usr/bin/env python
+"""
+ Parse Ruffus pipelines using old fashioned decorator syntax
+"""
+
+################################################################################
+#
+# parse_old_style_ruffus.py
+#
+#
+# Copyright (c) 25 February 2015 Leo Goodstadt
+# This is free software, and you are welcome to redistribute it
+# under the "MIT" license; See http://opensource.org/licenses/MIT for details.
+#
+#################################################################################
+
+import sys, os
+
+# Use import path from ~/python_modules
+if __name__ == '__main__':
+ sys.path.insert(0, os.path.expanduser("~lg/src/oss/ruffus"))
+ sys.path.append(os.path.expanduser("~lg/src/python_modules"))
+
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# options
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+
+if __name__ == '__main__':
+ import argparse
+
+ parser = argparse.ArgumentParser(description=__doc__)
+
+
+ common_group = parser.add_argument_group('Common arguments')
+ common_group.add_argument('--verbose', "-v", const=1, metavar="VERBOSITY", default=0, nargs='?', type= int,
+ help="Print more verbose messages for each additional verbose level.")
+ common_group.add_argument('--version', action='version', version='%(prog)s 1.0')
+ common_group.add_argument("-L", "--log_file", metavar="FILE", type=str,
+ help="Name and path of log file")
+
+ options = parser.parse_args()
+
+ if not options.log_file:
+ options.log_file = os.path.join("parse_old_style_ruffus.log")
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# imports
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+#from json import dumps
+#from collections import defaultdict
+import re
+
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# Functions
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# Logger
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+import logging
+import logging.handlers
+
+MESSAGE = 15
+logging.addLevelName(MESSAGE, "MESSAGE")
+
+def setup_std_logging (module_name, log_file, verbose):
+ """
+ set up logging using programme options
+ """
+
+ logger = logging.getLogger(module_name)
+
+
+ # We are interesting in all messages
+ logger.setLevel(logging.DEBUG)
+ has_handler = False
+
+ #
+ # log to file if that is specified
+ #
+ if log_file:
+ handler = logging.FileHandler(log_file, delay=False)
+ class stripped_down_formatter(logging.Formatter):
+ def format(self, record):
+ prefix = ""
+ if not hasattr(self, "first_used"):
+ self.first_used = True
+ prefix = "\n" + self.formatTime(record, "%Y-%m-%d")
+ prefix += " %(name)s\n" % record.__dict__
+ if record.levelname in ("INFO", "MESSAGE", "DEBUG"):
+ self._fmt = " %(asctime)s - %(message)s"
+ else:
+ self._fmt = " %(asctime)s - %(levelname)-7s - %(message)s"
+ return prefix + logging.Formatter.format(self, record)
+ handler.setFormatter(stripped_down_formatter("%(asctime)s - %(name)s - %(levelname)6s - %(message)s", "%H:%M:%S"))
+ handler.setLevel(MESSAGE)
+ logger.addHandler(handler)
+ has_handler = True
+
+ #
+ # log to stderr if verbose
+ #
+ if verbose:
+ stderrhandler = logging.StreamHandler(sys.stderr)
+ stderrhandler.setFormatter(logging.Formatter(" %(message)s"))
+ stderrhandler.setLevel(logging.DEBUG)
+ if log_file:
+ class debug_filter(logging.Filter):
+ """
+ Ignore INFO messages
+ """
+ def filter(self, record):
+ return logging.INFO != record.levelno
+ stderrhandler.addFilter(debug_filter())
+ logger.addHandler(stderrhandler)
+ has_handler = True
+
+ #
+ # no logging
+ #
+ if not has_handler:
+ class NullHandler(logging.Handler):
+ """
+ for when there is no logging
+ """
+ def emit(self, record):
+ pass
+ logger.addHandler(NullHandler())
+
+
+ return logger
+
+if __name__ == '__main__':
+
+ #
+ # set up log: name = script name sans extension
+ #
+ module_name = os.path.splitext(os.path.basename(sys.argv[0]))[0];
+ logger = setup_std_logging(module_name, options.log_file, options.verbose)
+
+
+ #
+ # log programme parameters
+ #
+ logger.info(" ".join(sys.argv))
+
+
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# Main logic
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+func_re = re.compile("^\s*def.*\(")
+def get_decorators(line_num, decorated_lines, all_lines):
+ for line in sys.stdin:
+ line_num += 1
+ mm = func_re.search(line)
+ if mm:
+ decorated_lines.append(line + "\n")
+ all_lines.append(line)
+ return line_num
+ else:
+ decorated_lines.append(line)
+
+ raise Exception("Unterminated decorators %s" % (decorated_lines,))
+
+
+decorator_re = re.compile("^\s*@")
+no_white_space_re = re.compile("^[^#\s]")
+if __name__ == '__main__':
+ line_num = 1
+ last_decorated_line_num = 1
+ pipeline_insertion_line_num = 1
+ decorated_lines = []
+ all_lines = []
+ for line in sys.stdin:
+ line_num += 1
+ # decorator start
+ if decorator_re.search(line):
+ decorated_lines.append(line)
+ line_num = get_decorators(line_num, decorated_lines, all_lines)
+ last_decorated_line_num = len(all_lines)
+ continue
+ if no_white_space_re.search(line):
+ if pipeline_insertion_line_num < last_decorated_line_num:
+ pipeline_insertion_line_num = len(all_lines)
+ all_lines.append(line)
+
+
+
+ for line in all_lines[0: pipeline_insertion_line_num]:
+ sys.stdout.write(line)
+ sys.stdout.write("\n" * 3)
+
+ sys.stdout.write('pipeline = Pipeline.pipelines["main"]\n\n')
+
+ for line in decorated_lines:
+ sys.stdout.write(line)
+ sys.stdout.write("\n" * 3)
+
+ for line in all_lines[pipeline_insertion_line_num:]:
+ sys.stdout.write(line)
+
+
+
+# for aa in active_if.py \
+# branching_dependencies.py \
+# cmdline.py \
+# collate.py \
+# empty_files_decorator.py \
+# exceptions.py \
+# file_name_parameters.py \
+# files_decorator.py \
+# filesre_combine.py \
+# filesre_split_and_combine.py \
+# follows_mkdir.py \
+# graphviz.py \
+# inputs_with_multiple_args_raising_exception.py \
+# job_completion_checksums.py \
+# job_history_with_exceptions.py \
+# mkdir.py \
+# N_x_M_and_collate.py \
+# pausing.py \
+# pipeline_printout_graph.py \
+# posttask_merge.py \
+# regex_error_messages.py \
+# ruffus_utility_parse_task_arguments.py \
+# ruffus_utility.py \
+# runtime_data.py \
+# softlink_uptodate.py \
+# split_and_combine.py \
+# split_regex_and_collate.py \
+# split_subdivide_checkpointing.py \
+# task_file_dependencies.py \
+# task_misc.py \
+# transform_add_inputs.py \
+# transform_inputs.py \
+# transform_with_no_re_matches.py \
+# tutorial7.py \
+# unicode_filenames.py \
+# verbosity.py; \
+# do echo test_$aa; ../parse_old_style_ruffus.py < test_$aa >| test_newstyle_$aa; done
+
diff --git a/ruffus/print_dependencies.py b/ruffus/print_dependencies.py
index 895f051..bf6eaeb 100644
--- a/ruffus/print_dependencies.py
+++ b/ruffus/print_dependencies.py
@@ -38,6 +38,11 @@ CNT_COLOUR_SCHEMES = 8
import types
+import sys
+try:
+ from StringIO import StringIO
+except:
+ from io import StringIO
from .adjacent_pairs_iterate import adjacent_pairs_iterate
from collections import defaultdict
def _get_name (node):
@@ -68,7 +73,8 @@ def attributes_to_str (attributes, name):
# if a label is specified, that overrides the node name
if "label" not in attributes:
- attributes["label"] = name
+ attributes["label"] = name.replace(" before ", "\\nbefore ").replace(", ", ",\n")
+
# remove any quotes
if attributes["label"][0] == '<':
@@ -378,7 +384,7 @@ def write_flowchart_in_dot_format( jobs_to_run,
up_to_date_jobs,
dag_violating_edges,
dag_violating_nodes,
- stream,
+ byte_stream,
target_jobs,
forced_to_run_jobs = [],
all_jobs = None,
@@ -433,7 +439,8 @@ def write_flowchart_in_dot_format( jobs_to_run,
"fillcolor"
Specifies legend colours
"""
- if user_colour_scheme == None:
+
+ if user_colour_scheme is None:
colour_scheme = get_default_colour_scheme()
else:
if "colour_scheme_index" in user_colour_scheme:
@@ -456,7 +463,7 @@ def write_flowchart_in_dot_format( jobs_to_run,
#
dag_violating_dependencies = set(dag_violating_edges)
-
+ stream = StringIO()
stream.write( 'digraph "%s"\n{\n' % pipeline_name)
stream.write( 'size="8,11";\n')
@@ -482,7 +489,7 @@ def write_flowchart_in_dot_format( jobs_to_run,
# all jobs should be specified
# this is a bad fall-back
# because there is no guarantee that we are printing what we want to print
- if all_jobs == None:
+ if all_jobs is None:
all_jobs = node.all_nodes
@@ -491,20 +498,20 @@ def write_flowchart_in_dot_format( jobs_to_run,
#
# defined duplicately in graph. Bad practice
#
- one_to_one = 0
- many_to_many = 1
- one_to_many = 2
- many_to_one = 3
+ _one_to_one = 0
+ _many_to_many = 1
+ _one_to_many = 2
+ _many_to_one = 3
for n in all_jobs:
attributes = dict()
attributes["shape"]="box3d"
#attributes["shape"] = "rect"
if hasattr(n, "single_multi_io"):
- if n.single_multi_io == one_to_many:
+ if n.single_multi_io == _one_to_many:
attributes["shape"] = "house"
attributes["peripheries"] = 2
- elif n.single_multi_io == many_to_one:
+ elif n.single_multi_io == _many_to_one:
attributes["shape"] = "invhouse"
attributes["height"] = 1.1
attributes["peripheries"] = 2
@@ -568,7 +575,7 @@ def write_flowchart_in_dot_format( jobs_to_run,
else:
get_dot_format_for_task_type ("Down stream" , attributes, colour_scheme, used_task_types)
delayed_task_strings.append('t%d' % n._node_index + attributes_to_str(attributes, _get_name(n)))
- for o in n.outward():
+ for o in n._get_inward():
delayed_task_strings.append('t%d -> t%d[color=%s, arrowtype=normal];\n' % (o._node_index, n._node_index, colour_scheme["Up-to-date"]["linecolor"]))
continue
@@ -585,7 +592,7 @@ def write_flowchart_in_dot_format( jobs_to_run,
# write edges
#
unconstrained = False
- for o in sorted(n.outward(), reverse=True, key = lambda x: x._node_index):
+ for o in sorted(n._get_inward(), reverse=True, key = lambda x: x._node_index):
#
# circularity violating DAG: highlight in red: should never be a constraint
# in drawing the graph
@@ -620,4 +627,8 @@ def write_flowchart_in_dot_format( jobs_to_run,
write_legend_key (stream, used_task_types, minimal_key_legend, colour_scheme)
stream.write("}\n")
+ ss = stream.getvalue().encode()
+ byte_stream.write(ss)
+
+ stream.close()
diff --git a/ruffus/proxy_logger.py b/ruffus/proxy_logger.py
index 317beed..fbf0032 100644
--- a/ruffus/proxy_logger.py
+++ b/ruffus/proxy_logger.py
@@ -293,6 +293,8 @@ class LoggerProxy(multiprocessing.managers.BaseProxy):
return self._callmethod('log', args, kwargs)
def __str__ (self):
return "<LoggingProxy>"
+ def __repr__ (self):
+ return 'LoggerProxy()'
#
# Register the setup_logger function as a proxy for setup_logger
diff --git a/ruffus/ruffus_exceptions.py b/ruffus/ruffus_exceptions.py
index f49fe2c..0f42e9a 100644
--- a/ruffus/ruffus_exceptions.py
+++ b/ruffus/ruffus_exceptions.py
@@ -61,7 +61,7 @@ class error_task(Exception):
Prefix with new lines for added emphasis
"""
# turn tasks names into 'def xxx(...): format
- task_names = "\n".join(t.get_task_name(True) for t in self.tasks)
+ task_names = "\n".join("task = %r" % t._name for t in self.tasks)
if len(self.main_msg):
return "\n\n" + self.main_msg + " for\n\n%s\n" % task_names
else:
@@ -196,6 +196,10 @@ class error_task_decorator_takes_no_args(error_task):
pass
class error_function_is_not_a_task(error_task):
pass
+class error_ambiguous_task(error_task):
+ pass
+class error_not_a_pipeline(error_task):
+ pass
class error_circular_dependencies(error_task):
pass
class error_not_a_directory(error_task):
@@ -212,6 +216,22 @@ class error_unescaped_regular_expression_forms(error_task):
pass
class error_checksum_level(error_task):
pass
+class error_missing_args(error_task):
+ pass
+class error_too_many_args(error_task):
+ pass
+class error_inputs_multiple_args(error_task):
+ pass
+class error_set_input(error_task):
+ pass
+class error_set_output(error_task):
+ pass
+class error_no_head_tasks(error_task):
+ pass
+class error_no_tail_tasks(error_task):
+ pass
+
+
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -227,11 +247,11 @@ if __name__ == '__main__':
# minimal task object to test exceptions
#
class task:
- class _task (object):
+ class Task (object):
"""
dummy task
"""
- action_mkdir = 1
+ _action_mkdir = 1
def __init__(self, _name, _action_type = 0):
self._action_type = _action_type
self._name = _name
@@ -249,10 +269,10 @@ if __name__ == '__main__':
"""
test
"""
- fake_task1 = task._task("task1")
- fake_task2 = task._task("task2")
- fake_mkdir_task3 = task._task("task3", task._task.action_mkdir)
- fake_mkdir_task4 = task._task("task4", task._task.action_mkdir)
+ fake_task1 = task.Task("task1")
+ fake_task2 = task.Task("task2")
+ fake_mkdir_task3 = task.Task("task3", task.Task._action_mkdir)
+ fake_mkdir_task4 = task.Task("task4", task.Task._action_mkdir)
e = error_task()
e.specify_task (fake_task1 , "Some message 0")
e.specify_task (fake_task2 , "Some message 1")
@@ -292,10 +312,10 @@ if __name__ == '__main__':
]
e = RethrownJobError(exception_data)
- fake_task1 = task._task("task1")
- fake_task2 = task._task("task2")
- fake_mkdir_task3 = task._task("task3", task._task.action_mkdir)
- fake_mkdir_task4 = task._task("task4", task._task.action_mkdir)
+ fake_task1 = task.Task("task1")
+ fake_task2 = task.Task("task2")
+ fake_mkdir_task3 = task.Task("task3", task.Task._action_mkdir)
+ fake_mkdir_task4 = task.Task("task4", task.Task._action_mkdir)
e.specify_task (fake_task1 , "Exceptions running jobs")
e.specify_task (fake_task2 , "Exceptions running jobs")
e.specify_task (fake_mkdir_task3, "Exceptions running jobs")
diff --git a/ruffus/ruffus_utility.py b/ruffus/ruffus_utility.py
index 6285cf9..ce00a19 100644
--- a/ruffus/ruffus_utility.py
+++ b/ruffus/ruffus_utility.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python
from __future__ import print_function
import sys
+import re
if sys.hexversion < 0x03000000:
from future_builtins import zip
################################################################################
@@ -105,6 +106,37 @@ CHECKSUM_FUNCTIONS_AND_PARAMS = 3 # also rerun when function parameters or f
CHECKSUM_REGENERATE = 2 # regenerate checksums
#_________________________________________________________________________________________
+
+# t_extra_inputs
+# namespaced enum
+
+#_________________________________________________________________________________________
+class t_extra_inputs:
+ (ADD_TO_INPUTS, REPLACE_INPUTS, KEEP_INPUTS, KEEP_OUTPUTS) = list(range(4))
+
+#_________________________________________________________________________________________
+
+# inputs
+
+#_________________________________________________________________________________________
+class inputs(object):
+ def __init__ (self, *args):
+ self.args = args
+ def __repr__ (self, *args):
+ return 'inputs%r' % (self.args,)
+
+#_________________________________________________________________________________________
+
+# add_inputs
+
+#_________________________________________________________________________________________
+class add_inputs(object):
+ def __init__ (self, *args):
+ self.args = args
+ def __repr__ (self, *args):
+ return 'add_inputs%r' % (self.args,)
+
+#_________________________________________________________________________________________
#
# get_default_checksum_level
#
@@ -139,7 +171,7 @@ def get_default_checksum_level ():
#
# check environmental variable is valid string
#
- if checksum_level == None:
+ if checksum_level is None:
raise error_checksum_level(("The environmental value "
"DEFAULT_RUFFUS_CHECKSUM_LEVEL should be: [0-3 | "
"CHECKSUM_FILE_TIMESTAMPS | "
@@ -274,7 +306,7 @@ def parameter_list_as_string (parameters):
E.g.
"""
- if parameters == None:
+ if parameters is None:
return ""
elif not isinstance(parameters, list):
raise Exception("Unexpected parameter list %s" % (parameters,))
@@ -442,11 +474,11 @@ def regex_match_str(test_str, compiled_regex):
compiled_regex = re.compile(compiled_regex)
mm = compiled_regex.search(test_str)
# Match failed
- if mm == None:
+ if mm is None:
return False
else:
# No capture
- if mm.lastindex == None:
+ if mm.lastindex is None:
return {0: mm.group(0)}
# Combined named and unnamed captures
else:
@@ -528,7 +560,7 @@ def path_decomposition_regex_match (test_str, compiled_regex):
# regular expression not specified
# just path
- if compiled_regex == None:
+ if compiled_regex is None:
return pp
rr = regex_match_str(test_str, compiled_regex)
@@ -556,7 +588,7 @@ def check_compiled_regexes (compiled_regexes, expected_num):
"""
check compiled_regexes are of the right type and number
"""
- if compiled_regexes == None:
+ if compiled_regexes is None:
return [None] * expected_num
if not isinstance(compiled_regexes, list):
@@ -819,22 +851,22 @@ def non_str_sequence (arg):
#_________________________________________________________________________________________
-# get_strings_in_nested_sequence_aux
+# get_strings_in_flattened_sequence_aux
# helper function for next function
#_________________________________________________________________________________________
-def get_strings_in_nested_sequence_aux(p, l = None):
+def get_strings_in_flattened_sequence_aux(p, l = None):
"""
Unravels arbitrarily nested sequence and returns lists of strings
"""
- if l == None:
+ if l is None:
l = []
if isinstance(p, path_str_type):
l.append(p)
elif non_str_sequence (p):
for pp in p:
- get_strings_in_nested_sequence_aux(pp, l)
+ get_strings_in_flattened_sequence_aux(pp, l)
return l
@@ -843,11 +875,11 @@ def get_strings_in_nested_sequence_aux(p, l = None):
# non_str_sequence
#_________________________________________________________________________________________
-def get_strings_in_nested_sequence (p, first_only = False):
+def get_strings_in_flattened_sequence (p):
"""
Traverses nested sequence and for each element, returns first string encountered
"""
- if p == None:
+ if p is None:
return []
#
@@ -859,48 +891,18 @@ def get_strings_in_nested_sequence (p, first_only = False):
#
# Get all strings flattened into list
#
- if not first_only:
- return get_strings_in_nested_sequence_aux(p)
+ return get_strings_in_flattened_sequence_aux(p)
- #
- # Get all first string in each element
- #
- elif non_str_sequence (p):
- filenames = []
- for pp in p:
- l = get_strings_in_nested_sequence_aux(pp)
- if len(l):
- filenames.append(l[0])
- return filenames
-
- return []
-
#_________________________________________________________________________________________
# get_first_string_in_nested_sequence
#_________________________________________________________________________________________
def get_first_string_in_nested_sequence (p):
- if p == None:
- return None
-
- #
- # string is returned as list of single string
- #
- if isinstance(p, path_str_type):
- return p
-
- #
- # Get all first string in each element
- #
- elif non_str_sequence (p):
- filenames = []
- for pp in p:
- l = get_strings_in_nested_sequence_aux(pp)
- if len(l):
- return l[0]
-
+ strings = get_strings_in_flattened_sequence (p)
+ if len(strings):
+ return strings[0]
return None
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -1019,15 +1021,16 @@ def get_nested_tasks_or_globs(p, treat_strings_as_tasks = False, runtime_data_na
#
# create storage if this is not a recursive call
#
- if globs == None:
- runtime_data_names, tasks, globs = set(), set(), set()
+ if globs is None:
+ runtime_data_names, tasks, globs = set(), list(), set()
#
# task function
#
if (isinstance(p, collections.Callable)):
- #if (type(p) == types.FunctionType):
- tasks.add(p)
+ tasks.append(p)
+ elif p.__class__.__name__ == 'Task' or p.__class__.__name__ == 'Pipeline':
+ tasks.append(p)
elif isinstance(p, runtime_parameter):
runtime_data_names.add(p)
@@ -1040,7 +1043,7 @@ def get_nested_tasks_or_globs(p, treat_strings_as_tasks = False, runtime_data_na
elif isinstance(p, path_str_type):
if treat_strings_as_tasks:
- tasks.add(p)
+ tasks.append(p)
elif is_glob(p):
globs.add(p)
@@ -1051,15 +1054,22 @@ def get_nested_tasks_or_globs(p, treat_strings_as_tasks = False, runtime_data_na
#_________________________________________________________________________________________
#
-# replace_func_names_with_tasks
+# replace_placeholders_with_tasks_in_input_params
#
#________________________________________________________________________________________
-def replace_func_names_with_tasks(p, func_or_name_to_task, treat_strings_as_tasks = False):
+def replace_placeholders_with_tasks_in_input_params(p, func_or_name_to_task, treat_strings_as_tasks = False):
"""
Replaces task functions or task name (strings) with the tasks they represent
+ Also replaces Tasks and Pipelines with the correct Tasks
func_or_name_to_task are a dictionary of function and task names to tasks
"""
+ if p.__class__.__name__ == 'Pipeline':
+ return func_or_name_to_task["PIPELINE=%s=PIPELINE" % p.name]
+
+ if p.__class__.__name__ == 'Task' and p in func_or_name_to_task:
+ return func_or_name_to_task[p]
+
#
# Expand globs or tasks as a list only if they are top level
#
@@ -1072,9 +1082,9 @@ def replace_func_names_with_tasks(p, func_or_name_to_task, treat_strings_as_task
#
if isinstance(p, output_from):
if len(p.args) == 1:
- return replace_func_names_with_tasks(p.args[0], func_or_name_to_task, True)
+ return replace_placeholders_with_tasks_in_input_params(p.args[0], func_or_name_to_task, True)
else:
- return [replace_func_names_with_tasks(pp, func_or_name_to_task, True) for pp in p.args]
+ return [replace_placeholders_with_tasks_in_input_params(pp, func_or_name_to_task, True) for pp in p.args]
#
# strings become tasks if treat_strings_as_tasks
@@ -1090,6 +1100,7 @@ def replace_func_names_with_tasks(p, func_or_name_to_task, treat_strings_as_task
if isinstance(p, dict):
return p
+
#
# Other sequences are recursed down
#
@@ -1105,13 +1116,13 @@ def replace_func_names_with_tasks(p, func_or_name_to_task, treat_strings_as_task
#
if isinstance(pp, output_from):
if len(pp.args) > 1:
- l.extend(tuple(replace_func_names_with_tasks(pp, func_or_name_to_task, True)))
+ l.extend(tuple(replace_placeholders_with_tasks_in_input_params(pp, func_or_name_to_task, True)))
elif len(pp.args) == 1:
- l.append(replace_func_names_with_tasks(pp.args[0], func_or_name_to_task, True))
+ l.append(replace_placeholders_with_tasks_in_input_params(pp.args[0], func_or_name_to_task, True))
# else len(pp.args) == 0 !! do nothing
else:
- l.append(replace_func_names_with_tasks(pp, func_or_name_to_task, treat_strings_as_tasks))
+ l.append(replace_placeholders_with_tasks_in_input_params(pp, func_or_name_to_task, treat_strings_as_tasks))
return type(p)(l)
#
@@ -1133,6 +1144,8 @@ def replace_func_names_with_tasks(p, func_or_name_to_task, treat_strings_as_task
class suffix(object):
def __init__ (self, *args):
self.args = args
+ def __repr__ (self, *args):
+ return 'suffix%r' % (self.args,)
#_________________________________________________________________________________________
@@ -1142,6 +1155,8 @@ class suffix(object):
class regex(object):
def __init__ (self, *args):
self.args = args
+ def __repr__ (self, *args):
+ return 'regex%r' % (self.args,)
#_________________________________________________________________________________________
@@ -1151,6 +1166,8 @@ class regex(object):
class formatter(object):
def __init__ (self, *args):
self.args = args
+ def __repr__ (self, *args):
+ return 'formatter%r' % (self.args,)
#_________________________________________________________________________________________
@@ -1330,7 +1347,7 @@ def check_files_io_parameters (enclosing_task, params, error_object):
if len(job_param) < 2:
raise error_object(enclosing_task, "Missing output files for job " +
ignore_unknown_encoder(job_param))
- #if len(get_strings_in_nested_sequence(job_param[0:2])) == 0:
+ #if len(get_strings_in_flattened_sequence(job_param[0:2])) == 0:
# raise error_object(enclosing_task, "Input or output file parameters should "
# "contain at least one or more file names strings. "
# "Consider using @parallel if you are not using files. " +
@@ -1367,7 +1384,7 @@ def expand_nested_tasks_or_globs(p, tasksglobs_to_filenames):
# Expand globs or tasks as a list only if they are top level
#
if ( (isinstance(p, path_str_type) and is_glob(p) and p in tasksglobs_to_filenames) or
- p.__class__.__name__ == '_task' or
+ p.__class__.__name__ == 'Task' or
isinstance(p, runtime_parameter) ):
return tasksglobs_to_filenames[p]
@@ -1385,7 +1402,7 @@ def expand_nested_tasks_or_globs(p, tasksglobs_to_filenames):
for pp in p:
if (isinstance(pp, path_str_type) and pp in tasksglobs_to_filenames):
l.extend(tasksglobs_to_filenames[pp])
- elif pp.__class__.__name__ == '_task' or isinstance(pp, runtime_parameter):
+ elif pp.__class__.__name__ == 'Task' or isinstance(pp, runtime_parameter):
files = tasksglobs_to_filenames[pp]
# task may have produced a single output: in which case append
if non_str_sequence(files):
@@ -1404,6 +1421,321 @@ def expand_nested_tasks_or_globs(p, tasksglobs_to_filenames):
+#_________________________________________________________________________________________
+
+# get_parsed_arguments_str_for_errors
+
+# helper funciton for parse_task_arguments()
+
+#_________________________________________________________________________________________
+def get_parsed_arguments_str_for_errors (task_description, bad_arg_str, unnamed_result_strs, named_result_strs):
+ """
+ Helper function for parse_task_arguments
+ Prints out offending argument (bad_arg_str) in the context of already parsed
+ arguments so that we can quickly figure out where the error is coming from
+ """
+ indent = task_description.find("(") + 1
+ parsed_arg_str = ", ".join(unnamed_result_strs + named_result_strs)
+ # make function names clearer in arg list
+ parsed_arg_str = re.sub(r"<function (\w+) at 0x[0-9a-f]+>", r"\1", parsed_arg_str)
+ return task_description % (parsed_arg_str + ", ...\n" +
+ # mark out problem
+ (" " * (indent-5 if indent - 5 > 0 else 0)) + "===> " +
+ bad_arg_str)
+
+
+
+#_________________________________________________________________________________________
+
+# parse_task_arguments
+
+#_________________________________________________________________________________________
+def parse_task_arguments ( orig_unnamed_arguments, orig_named_arguments, expected_arguments, task_description):
+ """
+ Parse arguments parsed into decorators or Pipeline.transform etc.
+ Special handling for optional arguments in the middle of argument list
+ 1) @product
+ can have (input, filter, input1, filter1, input2, filter2....)
+ 2) @transform, @subdivide, @collate, @product, @combinatorics which have
+ (..., [add_inputs(...)|inputs(...)],...)
+ or ([add_inputs=...|replace_inputs=...])
+ or ([add_inputs=add_inputs(...)|replace_inputs=inputs(...)])
+ Special handling for variable number of arguments at the end of the argument list
+ which all become "extras"
+
+ #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ #
+ # N.B. Missing non-mandatory arguments are returned as an empty list
+ #
+ #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+ """
+ results = {}
+ unnamed_arguments = list(orig_unnamed_arguments)
+ named_arguments = dict(orig_named_arguments)
+ # parsed results in string form for error messages
+ unnamed_result_strs = []
+ named_result_strs = []
+
+
+ #________________________________________________________________________________________
+ #
+ # parse_add_inputs()
+ #
+ #________________________________________________________________________________________
+ def parse_add_inputs_args(parsed_arg, input_type, arg_name, modify_inputs_mode, result_strs):
+ """
+ Parse arguments for add_inputs and replace_inputs, i.e. 'inputs()' and 'add_inputs()'
+ input_type =inputs|add_inputs
+ arg_name = replace_inputs | add_inputs
+ modify_inputs_mode = t_extra_inputs.REPLACE_INPUTS| t_extra_inputs.ADD_TO_INPUTS
+ """
+ results["modify_inputs_mode"] = modify_inputs_mode
+ if input_type == inputs:
+ # inputs() only takes a single argument. Throw error otherwise
+ if len(parsed_arg.args) != 1:
+ err_msg = "inputs() expects a single argument:\n%s" % (
+ get_parsed_arguments_str_for_errors(task_description, # bad arg in context of parsed
+ "%s%r" % (input_type.__name__, tuple(parsed_arg.args)),
+ unnamed_result_strs,
+ named_result_strs))
+ #print (err_msg, file=sys.stderr)
+ raise error_inputs_multiple_args(err_msg)
+
+ # unpack add_inputs / inputs and save results
+ results["modify_inputs"] = parsed_arg.args[0]
+ else:
+ results["modify_inputs"] = parsed_arg.args
+ result_strs.append("%s=%r" % (arg_name, parsed_arg.args))
+
+
+
+ #________________________________________________________________________________________
+ #
+ # check_argument_type
+ #
+ #________________________________________________________________________________________
+ def check_argument_type (arg_name, parsed_arg, argument_types):
+ """
+ check if parsed_arg is right type
+ """
+ if argument_types and not isinstance(parsed_arg, argument_types):
+ err_msg = ("The '%s' argument should be %s:\n%s" %
+ (arg_name, # argument name
+ " or ".join("%s" % tt.__name__ for tt in argument_types), # type names
+ get_parsed_arguments_str_for_errors(task_description, # bad arg in context of parsed
+ "%s = %r" % (arg_name, parsed_arg),
+ unnamed_result_strs, named_result_strs)))
+ #print (err_msg, file=sys.stderr)
+ raise TypeError(err_msg)
+
+ return parsed_arg
+ #________________________________________________________________________________________
+ #
+ # parse_argument
+ # helper function for parsing a single arguement
+ #________________________________________________________________________________________
+ def parse_argument (arg_name, expected_arguments, unnamed_arguments, named_arguments, results, task_description, mandatory, argument_types = None):
+ """
+ All missing, non-mandatory are empty list
+ """
+
+ # ignore if not on list
+ if not arg_name in expected_arguments:
+ return
+
+ #
+ # look among unnamed arguments first
+ #
+ if len(unnamed_arguments):
+ #
+ # check correct type
+ #
+ parsed_arg = check_argument_type (arg_name, unnamed_arguments[0], argument_types)
+ #
+ # Save parsed results
+ #
+ results[arg_name] = parsed_arg
+ unnamed_result_strs.append("%s=%r" % (arg_name, parsed_arg))
+ del unnamed_arguments[0]
+
+ #
+ # then named
+ #
+ elif arg_name in named_arguments:
+ #
+ # check correct type
+ #
+ parsed_arg = check_argument_type (arg_name, named_arguments[arg_name], argument_types)
+ #
+ # Save parsed results
+ #
+ results[arg_name] = parsed_arg
+ named_result_strs.append("%s=%r" % (arg_name, parsed_arg))
+ del named_arguments[arg_name]
+
+ #
+ # complain or ignore missing?
+ #
+ else:
+ if mandatory:
+ err_msg = "Missing '%s' argument:\n%s" % (
+ arg_name, # argument name
+ get_parsed_arguments_str_for_errors(task_description, # bad arg in
+ arg_name + " = ???",# context of parsed
+ unnamed_result_strs,
+ named_result_strs))
+ #print (err_msg, file=sys.stderr)
+ raise error_missing_args(err_msg)
+
+ #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ #
+ # N.B. Missing non-mandatory arguments are returned as an empty list
+ #
+ #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ else:
+ results[arg_name] = []
+
+ #
+ # Missing input is empty list
+ #
+ parse_argument ('input', expected_arguments, unnamed_arguments, named_arguments, results, task_description, mandatory = False)
+
+ #
+ # filter is mandatory if expected
+ #
+ parse_argument ('filter', expected_arguments, unnamed_arguments, named_arguments, results, task_description, mandatory = True, argument_types = (formatter, regex, suffix))
+
+ if "filter" in results:
+ if isinstance(results["filter"], suffix):
+ parse_argument ("output_dir", expected_arguments, [], named_arguments, results, task_description, mandatory = False)
+
+
+
+ #
+ # inputN
+ #
+ if 'inputN' in expected_arguments:
+ #
+ # put already parsed input and filter into the list
+ #
+ results["input"] = [results["input"]]
+ results["filter"] = [results["filter"]]
+ cnt_inputN = 1
+ #
+ # parse argument pairs at a time, so long as the second argument is a formatter
+ #
+ while len(unnamed_arguments) >= 2 and isinstance(unnamed_arguments[1], formatter):
+ filter_name = "filter%d" % (cnt_inputN + 1)
+ input_name = "input%d" % (cnt_inputN + 1)
+ unnamed_result_strs.append("%s=%r" % (input_name, unnamed_arguments[0]))
+ unnamed_result_strs.append("%s=%r" % (filter_name, unnamed_arguments[1]))
+ results["input"].append(unnamed_arguments[0])
+ results["filter"].append(unnamed_arguments[1])
+ cnt_inputN += 1
+ del unnamed_arguments[0:2]
+
+ #
+ # parse named arguments while there is a filter2, filter3 etc.
+ #
+ filter_name = "filter%d" % (cnt_inputN + 1)
+ input_name = "input%d" % (cnt_inputN + 1)
+ while filter_name in named_arguments:
+ results["filter"].append(named_arguments[filter_name])
+ named_result_strs.append("%s=%r" % (filter_name, named_arguments[filter_name]))
+ del named_arguments[filter_name]
+ # parse input2, input3 or leave blank as empty list
+ if input_name in named_arguments:
+ results["input"].append(named_arguments[input_name])
+ named_result_strs.append("%s=%r" % (input_name, named_arguments[input_name]))
+ del named_arguments[input_name]
+ else:
+ results["input"].append([])
+ cnt_inputN += 1
+ filter_name = "filter%d" % (cnt_inputN + 1)
+ input_name = "input%d" % (cnt_inputN + 1)
+
+
+ #
+ # tuple size is int and mandatory if exists
+ #
+ parse_argument ('tuple_size', expected_arguments, unnamed_arguments, named_arguments, results, task_description, mandatory = True, argument_types = (int,))
+
+ #
+ # add_inputs / inputs are optional
+ #
+ if 'modify_inputs' in expected_arguments:
+ results["modify_inputs_mode"] = t_extra_inputs.KEEP_INPUTS
+ results["modify_inputs"] = None
+ parse_add_inputs = ((inputs, "inputs", "replace_inputs", t_extra_inputs.REPLACE_INPUTS), (add_inputs, "add_inputs", "add_inputs", t_extra_inputs.ADD_TO_INPUTS))
+ if len(unnamed_arguments):
+ #
+ # Is add_inputs or inputs in unnamed arguments?
+ # Parse out contents and save in results["replace_inputs"] or results["add_inputs"]
+ #
+ for input_type, input_type_name, arg_name, modify_inputs_mode in parse_add_inputs:
+ parsed_arg = unnamed_arguments[0]
+ if isinstance(parsed_arg, input_type):
+ parse_add_inputs_args(parsed_arg, input_type, arg_name, modify_inputs_mode, unnamed_result_strs)
+ del unnamed_arguments[0]
+ break
+ #
+ # Otherwise is add_inputs or inputs in named arguments?
+ # Parse out contents only if necessary and save in results["replace_inputs"] or results["add_inputs"]
+ #
+ if results["modify_inputs_mode"] == t_extra_inputs.KEEP_INPUTS:
+ for input_type, input_type_name, arg_name, modify_inputs_mode in parse_add_inputs:
+ if arg_name in named_arguments:
+ parsed_arg = named_arguments[arg_name]
+ if isinstance(parsed_arg, input_type):
+ parse_add_inputs_args(parsed_arg, input_type, arg_name, modify_inputs_mode, named_result_strs)
+ else:
+ results["modify_inputs"] = parsed_arg
+ results["modify_inputs_mode"] = modify_inputs_mode
+ del named_arguments[arg_name]
+ break
+
+
+ #
+ # output is mandatory if exists
+ #
+ parse_argument ('output', expected_arguments, unnamed_arguments, named_arguments, results, task_description, mandatory = True)
+
+ #
+ # extras is mandatory if exists
+ #
+ if 'extras' in expected_arguments:
+ if len(unnamed_arguments):
+ # move list to results: remember python does shallow copies of lists
+ results['extras'] = unnamed_arguments
+ unnamed_result_strs.append("%s=%r" % ("extras", unnamed_arguments))
+ unnamed_arguments = []
+ #del unnamed_arguments[:]
+ elif 'extras' in named_arguments:
+ results['extras' ] = named_arguments['extras' ]
+ named_result_strs.append("%s=%r" % ("extras", named_arguments['extras' ]))
+ del named_arguments['extras' ]
+ else:
+ results['extras' ] = []
+
+
+ if len(unnamed_arguments):
+ err_msg = ("Too many unnamed arguments leftover:\n%s" %
+ get_parsed_arguments_str_for_errors(task_description, # bad arg in context of parsed
+ (", ".join(("%r" % a) for a in unnamed_arguments)),
+ unnamed_result_strs, named_result_strs))
+ #print (err_msg, file=sys.stderr)
+ raise error_too_many_args(err_msg)
+ if len(named_arguments):
+ err_msg = ("Duplicate, conflicting or unrecognised arguments:\n%s" %
+ get_parsed_arguments_str_for_errors(task_description, # bad arg in context of parsed
+ ", ".join("%s=%r" %(k, v) for k,v in named_arguments.items()),
+ unnamed_result_strs, named_result_strs))
+ #print (err_msg, file=sys.stderr)
+ raise error_too_many_args(err_msg)
+
+
+ return results
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -1417,6 +1749,8 @@ class combine(object):
class output_from(object):
def __init__ (self, *args):
self.args = args
+ def __repr__ (self, *args):
+ return 'output_from%r' % (self.args,)
class runtime_parameter(object):
def __init__ (self, *args):
diff --git a/ruffus/ruffus_version.py b/ruffus/ruffus_version.py
index 18f2197..7725cbb 100755
--- a/ruffus/ruffus_version.py
+++ b/ruffus/ruffus_version.py
@@ -24,4 +24,4 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#################################################################################
-__version='2.5'
+__version='2.6.2'
diff --git a/ruffus/task.py b/ruffus/task.py
index 09ac09c..85900df 100644
--- a/ruffus/task.py
+++ b/ruffus/task.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from __future__ import print_function
import sys
-#import signal
+# import signal
if sys.hexversion < 0x03000000:
from future_builtins import zip, map
################################################################################
@@ -79,21 +79,21 @@ Running the pipeline
"""
-
-
-import os,sys,copy, multiprocessing
-#from collections import namedtuple
+import os
+import sys
+import copy
+import multiprocessing
import collections
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
# imports
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
import logging
import re
-from collections import defaultdict
+from collections import defaultdict, deque
from multiprocessing import Pool
from multiprocessing.pool import ThreadPool
import traceback
@@ -117,13 +117,13 @@ from . import dbdict
if __name__ == '__main__':
import sys
- sys.path.insert(0,".")
+ sys.path.insert(0, ".")
from .graph import *
from .print_dependencies import *
-from .ruffus_exceptions import *
+from .ruffus_exceptions import *
from .ruffus_utility import *
-from .file_name_parameters import *
+from .file_name_parameters import *
if sys.hexversion >= 0x03000000:
# everything is unicode in python3
@@ -147,83 +147,110 @@ if sys.hexversion >= 0x03000000:
else:
import Queue as queue
+
class Ruffus_Keyboard_Interrupt_Exception (Exception):
pass
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
#
# light weight logging objects
#
#
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
+
+
class t_black_hole_logger:
+
"""
Does nothing!
"""
- def info (self, message, *args, **kwargs):
+
+ def info(self, message, *args, **kwargs):
pass
- def debug (self, message, *args, **kwargs):
+
+ def debug(self, message, *args, **kwargs):
pass
- def warning (self, message, *args, **kwargs):
+
+ def warning(self, message, *args, **kwargs):
pass
- def error (self, message, *args, **kwargs):
+
+ def error(self, message, *args, **kwargs):
pass
class t_stderr_logger:
+
"""
Everything to stderr
"""
- def __init__ (self):
+
+ def __init__(self):
self.unique_prefix = ""
- def add_unique_prefix (self):
+
+ def add_unique_prefix(self):
import random
random.seed()
- self.unique_prefix= str(random.randint(0,1000)) + " "
- def info (self, message):
+ self.unique_prefix = str(random.randint(0, 1000)) + " "
+
+ def info(self, message):
sys.stderr.write(self.unique_prefix + message + "\n")
- def warning (self, message):
+
+ def warning(self, message):
sys.stderr.write("\n\n" + self.unique_prefix + "WARNING:\n " + message + "\n\n")
- def error (self, message):
+
+ def error(self, message):
sys.stderr.write("\n\n" + self.unique_prefix + "ERROR:\n " + message + "\n\n")
- def debug (self, message):
+
+ def debug(self, message):
sys.stderr.write(self.unique_prefix + message + "\n")
+
class t_stream_logger:
+
"""
Everything to stderr
"""
- def __init__ (self, stream):
+
+ def __init__(self, stream):
self.stream = stream
- def info (self, message):
+
+ def info(self, message):
self.stream.write(message + "\n")
- def warning (self, message):
+
+ def warning(self, message):
sys.stream.write("\n\nWARNING:\n " + message + "\n\n")
- def error (self, message):
+
+ def error(self, message):
sys.stream.write("\n\nERROR:\n " + message + "\n\n")
- def debug (self, message):
+
+ def debug(self, message):
self.stream.write(message + "\n")
black_hole_logger = t_black_hole_logger()
-stderr_logger = t_stderr_logger()
+stderr_logger = t_stderr_logger()
+
class t_verbose_logger:
- def __init__ (self, verbose, verbose_abbreviated_path, logger, runtime_data):
+
+ def __init__(self, verbose, verbose_abbreviated_path, logger, runtime_data):
self.verbose = verbose
self.logger = logger
self.runtime_data = runtime_data
self.verbose_abbreviated_path = verbose_abbreviated_path
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
#
# logging helper function
#
-#________________________________________________________________________________________
-def log_at_level (logger, message_level, verbose_level, msg):
+# _____________________________________________________________________________
+
+
+def log_at_level(logger, message_level, verbose_level, msg):
"""
writes to log if message_level > verbose level
- Returns anything written in case we might want to drop down and output at a lower log level
+ Returns anything written in case we might want to drop down and output at a
+ lower log level
"""
if message_level <= verbose_level:
logger.info(msg)
@@ -231,26 +258,20 @@ def log_at_level (logger, message_level, verbose_level, msg):
return False
-
-
-
-
-
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
# queue management objects
# inserted into queue like job parameters to control multi-processing queue
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
# fake parameters to signal in queue
class all_tasks_complete:
pass
+
class waiting_for_more_tasks_to_complete:
pass
@@ -258,8 +279,8 @@ class waiting_for_more_tasks_to_complete:
#
# synchronisation data
#
-#SyncManager()
-#syncmanager.start()
+# SyncManager()
+# syncmanager.start()
#
# do nothing semaphore
@@ -269,27 +290,21 @@ def do_nothing_semaphore():
yield
-
# EXTRA pipeline_run DEBUGGING
EXTRA_PIPELINERUN_DEBUGGING = False
-
-
-
-
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
# task_decorator
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
class task_decorator(object):
+
"""
- Adds task to the "pipeline_task" attribute of this function but
- otherwise leaves function untouched
+ Forwards to functions within Task
"""
+
def __init__(self, *decoratorArgs, **decoratorNamedArgs):
"""
saves decorator arguments
@@ -297,27 +312,28 @@ class task_decorator(object):
self.args = decoratorArgs
self.named_args = decoratorNamedArgs
- def __call__(self, func):
+ def __call__(self, task_func):
"""
calls func in task with the same name as the class
"""
- # add task as attribute of this function
- if not hasattr(func, "pipeline_task"):
- func.pipeline_task = _task.create_task(func)
-
+ # add task to main pipeline
+ # check for duplicate tasks inside _create_task
+ task = main_pipeline._create_task(task_func)
# call the method called
- # "task.task_decorator"
- # where "task_decorator" is the name of this class
- decorator_function_name = "task_" + self.__class__.__name__
- task_decorator_function = getattr(func.pipeline_task, decorator_function_name)
- task_decorator_function(self.args, **self.named_args)
-
+ # task.decorator_xxxx
+ # where xxxx = transform subdivide etc
+ task_decorator_function = getattr(task, "_decorator_" + self.__class__.__name__)
+ task.created_via_decorator = True
+ # create empty placeholder with the args %s actually inside the task function
+ task.description_with_args_placeholder = task._get_decorated_function(
+ ).replace("...", "%s", 1)
+ task_decorator_function(*self.args, **self.named_args)
#
# don't change the function so we can call it unaltered
#
- return func
+ return task_func
#
@@ -326,35 +342,43 @@ class task_decorator(object):
class follows(task_decorator):
pass
+
class files(task_decorator):
pass
-
-
-
-
#
# Core
#
class split(task_decorator):
pass
+
class transform(task_decorator):
pass
+
class subdivide(task_decorator):
+
+ """
+ Splits a each set of input files into multiple output file names,
+ where the number of output files may not be known beforehand.
+ """
pass
+
class originate(task_decorator):
pass
+
class merge(task_decorator):
pass
+
class posttask(task_decorator):
pass
+
class jobs_limit(task_decorator):
pass
@@ -365,180 +389,180 @@ class jobs_limit(task_decorator):
class collate(task_decorator):
pass
+
class active_if(task_decorator):
pass
#
# Esoteric
#
+
+
class check_if_uptodate(task_decorator):
pass
+
class parallel(task_decorator):
pass
+
class graphviz(task_decorator):
pass
#
# Obsolete
#
-class files_re(task_decorator):
- pass
-
+class files_re(task_decorator):
+ pass
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
# indicator objects
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-#_________________________________________________________________________________________
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
+# _____________________________________________________________________________
# mkdir
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
class mkdir(task_decorator):
- #def __init__ (self, *args):
+ # def __init__ (self, *args):
# self.args = args
pass
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
# touch_file
-#_________________________________________________________________________________________
-class touch_file(object):
- def __init__ (self, *args):
- self.args = args
-
-
+# _____________________________________________________________________________
-#_________________________________________________________________________________________
-# inputs
+class touch_file(object):
-#_________________________________________________________________________________________
-class inputs(object):
- def __init__ (self, *args):
+ def __init__(self, *args):
self.args = args
-#_________________________________________________________________________________________
-
-# add_inputs
-
-#_________________________________________________________________________________________
-class add_inputs(object):
- def __init__ (self, *args):
- self.args = args
-#8888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
# job descriptors
# given parameters, returns strings describing job
# First returned parameter is string in strong form
-# Second returned parameter is a list of strings for input, output and extra parameters
+# Second returned parameter is a list of strings for input,
+# output and extra parameters
# intended to be reformatted with indentation
# main use in error logging
-#8888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-def generic_job_descriptor (param, verbose_abbreviated_path, runtime_data):
- if param in ([], None):
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
+def generic_job_descriptor(unglobbed_params, verbose_abbreviated_path, runtime_data):
+ if unglobbed_params in ([], None):
m = "Job"
else:
- m = "Job = %s" % ignore_unknown_encoder(param)
-
+ m = "Job = %s" % ignore_unknown_encoder(unglobbed_params)
return m, [m]
-def io_files_job_descriptor (param, verbose_abbreviated_path, runtime_data):
- extra_param = ", " + shorten_filenames_encoder(param[2:], verbose_abbreviated_path)[1:-1] if len(param) > 2 else ""
- out_param = shorten_filenames_encoder(param[1], verbose_abbreviated_path) if len(param) > 1 else "??"
- in_param = shorten_filenames_encoder(param[0], verbose_abbreviated_path) if len(param) > 0 else "??"
+
+def io_files_job_descriptor(unglobbed_params, verbose_abbreviated_path, runtime_data):
+ extra_param = ", " + shorten_filenames_encoder(unglobbed_params[2:], verbose_abbreviated_path)[1:-1] \
+ if len(unglobbed_params) > 2 else ""
+ out_param = shorten_filenames_encoder(unglobbed_params[1], verbose_abbreviated_path) \
+ if len(unglobbed_params) > 1 else "??"
+ in_param = shorten_filenames_encoder(unglobbed_params[0], verbose_abbreviated_path) \
+ if len(unglobbed_params) > 0 else "??"
return ("Job = [%s -> %s%s]" % (in_param, out_param, extra_param),
["Job = [%s" % in_param, "-> " + out_param + extra_param + "]"])
-def io_files_one_to_many_job_descriptor (param, verbose_abbreviated_path, runtime_data):
+def io_files_one_to_many_job_descriptor(unglobbed_params, verbose_abbreviated_path, runtime_data):
- extra_param = ", " + shorten_filenames_encoder(param[2:], verbose_abbreviated_path)[1:-1] if len(param) > 2 else ""
- out_param = shorten_filenames_encoder(param[1], verbose_abbreviated_path) if len(param) > 1 else "??"
- in_param = shorten_filenames_encoder(param[0], verbose_abbreviated_path) if len(param) > 0 else "??"
+ extra_param = ", " + shorten_filenames_encoder(unglobbed_params[2:], verbose_abbreviated_path)[1:-1] \
+ if len(unglobbed_params) > 2 else ""
+ out_param = shorten_filenames_encoder(unglobbed_params[1], verbose_abbreviated_path) \
+ if len(unglobbed_params) > 1 else "??"
+ in_param = shorten_filenames_encoder(unglobbed_params[0], verbose_abbreviated_path) \
+ if len(unglobbed_params) > 0 else "??"
# start with input parameter
ret_params = ["Job = [%s" % in_param]
# add output parameter to list,
# processing one by one if multiple output parameters
- if len(param) > 1:
- if isinstance(param[1], (list, tuple)):
- ret_params.extend("-> " + shorten_filenames_encoder(p, verbose_abbreviated_path) for p in param[1])
+ if len(unglobbed_params) > 1:
+ if isinstance(unglobbed_params[1], (list, tuple)):
+ ret_params.extend(
+ "-> " + shorten_filenames_encoder(p, verbose_abbreviated_path) for p in unglobbed_params[1])
else:
ret_params.append("-> " + out_param)
# add extra
- if len(param) > 2 :
- ret_params.append(" , " + shorten_filenames_encoder(param[2:], verbose_abbreviated_path)[1:-1])
+ if len(unglobbed_params) > 2:
+ ret_params.append(
+ " , " + shorten_filenames_encoder(unglobbed_params[2:], verbose_abbreviated_path)[1:-1])
# add closing bracket
- ret_params[-1] +="]"
+ ret_params[-1] += "]"
return ("Job = [%s -> %s%s]" % (in_param, out_param, extra_param), ret_params)
-def mkdir_job_descriptor (param, verbose_abbreviated_path, runtime_data):
+def mkdir_job_descriptor(unglobbed_params, verbose_abbreviated_path, runtime_data):
# input, output and parameters
- if len(param) == 1:
- m = "Make directories %s" % (shorten_filenames_encoder(param[0], verbose_abbreviated_path))
- elif len(param) == 2:
- m = "Make directories %s" % (shorten_filenames_encoder(param[1], verbose_abbreviated_path))
+ if len(unglobbed_params) == 1:
+ m = "Make directories %s" % (shorten_filenames_encoder(unglobbed_params[0], verbose_abbreviated_path))
+ elif len(unglobbed_params) == 2:
+ m = "Make directories %s" % (shorten_filenames_encoder(unglobbed_params[1], verbose_abbreviated_path))
else:
return [], []
return m, [m]
-#8888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
# job wrappers
# registers files/directories for cleanup
-#8888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-#_________________________________________________________________________________________
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
+# _____________________________________________________________________________
# generic job wrapper
-#_________________________________________________________________________________________
-def job_wrapper_generic(param, user_defined_work_func, register_cleanup, touch_files_only):
+# _____________________________________________________________________________
+def job_wrapper_generic(params, user_defined_work_func, register_cleanup, touch_files_only):
"""
run func
"""
assert(user_defined_work_func)
- return user_defined_work_func(*param)
+ return user_defined_work_func(*params)
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
# job wrapper for all that deal with i/o files
-#_________________________________________________________________________________________
-def job_wrapper_io_files(param, user_defined_work_func, register_cleanup, touch_files_only, output_files_only = False):
+# _____________________________________________________________________________
+
+
+def job_wrapper_io_files(params, user_defined_work_func, register_cleanup, touch_files_only,
+ output_files_only=False):
"""
run func on any i/o if not up to date
"""
assert(user_defined_work_func)
- i,o = param[0:2]
+ i, o = params[0:2]
if touch_files_only == 0:
# @originate only uses output files
if output_files_only:
- ret_val = user_defined_work_func(*(param[1:]))
+ ret_val = user_defined_work_func(*(params[1:]))
# all other decorators
else:
try:
- ret_val = user_defined_work_func(*param)
+ ret_val = user_defined_work_func(*params)
# EXTRA pipeline_run DEBUGGING
if EXTRA_PIPELINERUN_DEBUGGING:
sys.stderr.write("w" * 36 + "[[ task() done ]]" + "w" * 27 + "\n")
@@ -546,91 +570,105 @@ def job_wrapper_io_files(param, user_defined_work_func, register_cleanup, touch_
# Reraise KeyboardInterrupt as a normal Exception
# EXTRA pipeline_run DEBUGGING
if EXTRA_PIPELINERUN_DEBUGGING:
- sys.stderr.write("E" * 36 + "[[ KeyboardInterrupt from task() ]]" + "E" * 9 + "\n")
+ sys.stderr.write("E" * 36 + "[[ KeyboardInterrupt from task() ]]" +
+ "E" * 9 + "\n")
raise Ruffus_Keyboard_Interrupt_Exception("KeyboardInterrupt")
except:
- #sys.stderr.write("?? %s ??" % (tuple(param),))
+ # sys.stderr.write("?? %s ??" % (tuple(params),))
raise
elif touch_files_only == 1:
- #job_history = dbdict.open(RUFFUS_HISTORY_FILE, picklevalues=True)
+ # job_history = dbdict.open(RUFFUS_HISTORY_FILE, picklevalues=True)
+
+ #
+ # Do not touch any output files which are the same as any input
+ # i.e. which are just being passed through
+ #
+ # list of input files
+ real_input_file_names = set()
+ for f in get_strings_in_flattened_sequence(i):
+ real_input_file_names.add(os.path.realpath(f))
#
# touch files only
#
- for f in get_strings_in_nested_sequence(o):
+ for f in get_strings_in_flattened_sequence(o):
+
+ if os.path.realpath(f) in real_input_file_names:
+ continue
+
#
# race condition still possible...
#
with open(f, 'a') as ff:
os.utime(f, None)
- #if not os.path.exists(f):
+ # if not os.path.exists(f):
# open(f, 'w')
# mtime = os.path.getmtime(f)
- #else:
+ # else:
# os.utime(f, None)
# mtime = os.path.getmtime(f)
-
- #chksum = JobHistoryChecksum(f, mtime, param[2:], user_defined_work_func.pipeline_task)
- #job_history[f] = chksum # update file times and job details in history
-
-
+ # job_history[f] = chksum # update file times and job details in
+ # history
#
# register strings in output file for cleanup
#
- for f in get_strings_in_nested_sequence(o):
+ for f in get_strings_in_flattened_sequence(o):
register_cleanup(f, "file")
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
# job wrapper for all that only deals with output files
-#_________________________________________________________________________________________
-def job_wrapper_output_files(param, user_defined_work_func, register_cleanup, touch_files_only):
+# _____________________________________________________________________________
+def job_wrapper_output_files(params, user_defined_work_func, register_cleanup, touch_files_only):
"""
run func on any output file if not up to date
"""
- job_wrapper_io_files(param, user_defined_work_func, register_cleanup, touch_files_only, output_files_only = True)
+ job_wrapper_io_files(params, user_defined_work_func, register_cleanup, touch_files_only,
+ output_files_only=True)
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
# job wrapper for mkdir
-#_________________________________________________________________________________________
-def job_wrapper_mkdir(param, user_defined_work_func, register_cleanup, touch_files_only):
+# _____________________________________________________________________________
+def job_wrapper_mkdir(params, user_defined_work_func, register_cleanup, touch_files_only):
"""
- make directories if not exists
+ Make missing directories including any intermediate directories on the specified path(s)
"""
#
- # Just in case, swallow file exist errors because some other makedirs might be subpath
- # of this directory
+ # Just in case, swallow file exist errors because some other makedirs
+ # might be subpath of this directory
# Should not be necessary because of "sorted" in task_mkdir
#
#
- if len(param) == 1:
- dirs = param[0]
+ if len(params) == 1:
+ dirs = params[0]
- # if there are two parameters, they are i/o, and the directories to be created are the output
- elif len(param) == 2:
- dirs = param[1]
+ # if there are two parameters, they are i/o, and the directories to be
+ # created are the output
+ elif len(params) >= 2:
+ dirs = params[1]
else:
- raise Exception("Wrong number of arguments in mkdir check %s" % (param,))
+ raise Exception("No arguments in mkdir check %s" % (params,))
# get all file names in flat list
- dirs = get_strings_in_nested_sequence (dirs)
+ dirs = get_strings_in_flattened_sequence(dirs)
for d in dirs:
try:
- os.makedirs(d) # Please email the authors if an uncaught exception is raised here
+ # Please email the authors if an uncaught exception is raised here
+ os.makedirs(d)
register_cleanup(d, "makedirs")
except:
#
# ignore exception if
- # exception == OSError + "File exists" or // Linux
- # exception == WindowsError + "file already exists" // Windows
+ # exception == OSError + "File exists" or // Linux
+ # exception == WindowsError + "file already exists" // Windows
# Are other exceptions raised by other OS?
#
#
@@ -644,76 +682,43 @@ def job_wrapper_mkdir(param, user_defined_work_func, register_cleanup, touch_fil
raise
# changed for compatibility with python 3.x
- #except OSError, e:
+ # except OSError, e:
# if "File exists" not in e:
# raise
-JOB_ERROR = 0
+JOB_ERROR = 0
JOB_SIGNALLED_BREAK = 1
-JOB_UP_TO_DATE = 2
-JOB_COMPLETED = 3
+JOB_UP_TO_DATE = 2
+JOB_COMPLETED = 3
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
# t_job_result
# Previously a collections.namedtuple (introduced in python 2.6)
# Now using implementation from running
-# t_job_result = namedtuple('t_job_result', 'task_name state job_name return_value exception', verbose =1)
+# t_job_result = namedtuple('t_job_result',
+# 'task_name state job_name return_value exception', verbose =1)
# for compatibility with python 2.5
-#_________________________________________________________________________________________
-t_job_result = namedtuple('t_job_result', 'task_name state job_name return_value exception params', verbose =0)
-#class t_job_result(tuple):
-# 't_job_result(task_name, state, job_name, return_value, exception, params)'
-#
-# __slots__ = ()
-#
-# fields = ('task_name', 'state', 'job_name', 'return_value', 'exception', 'params')
-#
-# def __new__(cls, task_name, state, job_name, return_value, exception, params):
-# return tuple.__new__(cls, (task_name, state, job_name, return_value, exception, params))
-#
-# @classmethod
-# def make(cls, iterable, new=tuple.__new__, len=len):
-# 'Make a new t_job_result object from a sequence or iterable'
-# result = new(cls, iterable)
-# if len(result) != 6:
-# raise TypeError('Expected 6 arguments, got %d' % len(result))
-# return result
-#
-# def __repr__(self):
-# return 't_job_result(task_name=%r, state=%r, job_name=%r, return_value=%r, exception=%r, params=%r)' % self
-#
-# def asdict(t):
-# 'Return a new dict which maps field names to their values'
-# return {'task_name': t[0], 'state': t[1], 'job_name': t[2], 'return_value': t[3], 'exception': t[4], 'params':t[5]}
-#
-# def replace(self, **kwds):
-# 'Return a new t_job_result object replacing specified fields with new values'
-# result = self.make(list(map(kwds.pop, ('task_name', 'state', 'job_name', 'return_value', 'exception', 'params'), self)))
-# if kwds:
-# raise ValueError('Got unexpected field names: %r' % list(kwds.keys()))
-# return result
-#
-# def __getnewargs__(self):
-# return tuple(self)
-#
-# task_name = property(itemgetter(0))
-# state = property(itemgetter(1))
-# job_name = property(itemgetter(2))
-# return_value= property(itemgetter(3))
-# exception = property(itemgetter(4))
-# params = property(itemgetter(5))
-
+# _____________________________________________________________________________
+t_job_result = namedtuple('t_job_result',
+ 'task_name '
+ 'node_index state '
+ 'job_name '
+ 'return_value '
+ 'exception '
+ 'params '
+ 'unglobbed_params ',
+ verbose=0)
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
# multiprocess_callback
#
-#_________________________________________________________________________________________
-def run_pooled_job_without_exceptions (process_parameters):
+# _____________________________________________________________________________
+def run_pooled_job_without_exceptions(process_parameters):
"""
handles running jobs in parallel
Make sure exceptions are caught here:
@@ -721,18 +726,18 @@ def run_pooled_job_without_exceptions (process_parameters):
return any exceptions which will be rethrown at the other end:
See RethrownJobError / run_all_jobs_in_task
"""
- #signal.signal(signal.SIGINT, signal.SIG_IGN)
- (param, task_name, job_name, job_wrapper, user_defined_work_func,
- job_limit_semaphore, death_event, touch_files_only) = process_parameters
-
- ##job_history = dbdict.open(RUFFUS_HISTORY_FILE, picklevalues=True)
- ##outfile = param[1] if len(param) > 1 else None # mkdir has no output
- ##if not isinstance(outfile, list):
- ## outfile = [outfile]
- ##for o in outfile:
- ## job_history.pop(o, None) # remove outfile from history if it exists
-
- if job_limit_semaphore == None:
+ # signal.signal(signal.SIGINT, signal.SIG_IGN)
+ (params, unglobbed_params, task_name, node_index, job_name, job_wrapper, user_defined_work_func,
+ job_limit_semaphore, death_event, touch_files_only) = process_parameters
+
+ # #job_history = dbdict.open(RUFFUS_HISTORY_FILE, picklevalues=True)
+ # outfile = params[1] if len(params) > 1 else None # mkdir has no output
+ # if not isinstance(outfile, list):
+ # # outfile = [outfile]
+ # for o in outfile:
+ # job_history.pop(o, None) # remove outfile from history if it exists
+
+ if job_limit_semaphore is None:
job_limit_semaphore = do_nothing_semaphore()
try:
@@ -740,19 +745,22 @@ def run_pooled_job_without_exceptions (process_parameters):
# EXTRA pipeline_run DEBUGGING
if EXTRA_PIPELINERUN_DEBUGGING:
sys.stderr.write(">" * 36 + "[[ job_wrapper ]]" + ">" * 27 + "\n")
- return_value = job_wrapper(param, user_defined_work_func, register_cleanup, touch_files_only)
+ return_value = job_wrapper(params, user_defined_work_func,
+ register_cleanup, touch_files_only)
#
# ensure one second between jobs
#
- #if one_second_per_job:
+ # if one_second_per_job:
# time.sleep(1.01)
# EXTRA pipeline_run DEBUGGING
if EXTRA_PIPELINERUN_DEBUGGING:
sys.stderr.write("<" * 36 + "[[ job_wrapper done ]]" + "<" * 22 + "\n")
- return t_job_result(task_name, JOB_COMPLETED, job_name, return_value, None, param)
+ return t_job_result(task_name, node_index, JOB_COMPLETED, job_name, return_value, None,
+ params, unglobbed_params)
except KeyboardInterrupt as e:
- # Reraise KeyboardInterrupt as a normal Exception. Should never be necessary here
+ # Reraise KeyboardInterrupt as a normal Exception.
+ # Should never be necessary here
# EXTRA pipeline_run DEBUGGING
if EXTRA_PIPELINERUN_DEBUGGING:
sys.stderr.write("E" * 36 + "[[ KeyboardInterrupt ]]" + "E" * 21 + "\n")
@@ -764,11 +772,12 @@ def run_pooled_job_without_exceptions (process_parameters):
sys.stderr.write("E" * 36 + "[[ Other Interrupt ]]" + "E" * 23 + "\n")
# Wrap up one or more exceptions rethrown across process boundaries
#
- # See multiprocessor.Server.handle_request/serve_client for an analogous function
+ # See multiprocessor.Server.handle_request/serve_client for an
+ # analogous function
exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
- exception_stack = traceback.format_exc()
- exception_name = exceptionType.__module__ + '.' + exceptionType.__name__
- exception_value = str(exceptionValue)
+ exception_stack = traceback.format_exc()
+ exception_name = exceptionType.__module__ + '.' + exceptionType.__name__
+ exception_value = str(exceptionValue)
if len(exception_value):
exception_value = "(%s)" % exception_value
@@ -779,396 +788,1410 @@ def run_pooled_job_without_exceptions (process_parameters):
job_state = JOB_SIGNALLED_BREAK
else:
job_state = JOB_ERROR
- return t_job_result(task_name, job_state, job_name, None,
+ return t_job_result(task_name, node_index, job_state, job_name, None,
[task_name,
job_name,
exception_name,
exception_value,
- exception_stack], param)
-
+ exception_stack], params, unglobbed_params)
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
# Helper function
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
# register_cleanup
# to do
-#_________________________________________________________________________________________
-def register_cleanup (file_name, operation):
+# _____________________________________________________________________________
+def register_cleanup(file_name, operation):
pass
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# _____________________________________________________________________________
-# _task
+# pipeline functions only have "name" as a named parameter
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-class _task (node):
- """
- pipeline task
- """
-
- action_names = ["unspecified",
- "task",
- "task_files_re",
- "task_split",
- "task_merge",
- "task_transform",
- "task_collate",
- "task_files_func",
- "task_files",
- "task_mkdir",
- "task_parallel",
- "task_active_if",
- "task_product",
- "task_permutations",
- "task_combinations",
- "task_combinations_with_replacement",
- "task_subdivide",
- "task_originate",
- "task_graphviz",
- ]
- action_unspecified = 0
- action_task = 1
- action_task_files_re = 2
- action_task_split = 3
- action_task_merge = 4
- action_task_transform = 5
- action_task_collate = 6
- action_task_files_func = 7
- action_task_files = 8
- action_mkdir = 9
- action_parallel = 10
- action_active_if = 11
- action_task_product = 12
- action_task_permutations = 13
- action_task_combinations = 14
- action_task_combinations_with_replacement = 15
- action_task_subdivide = 16
- action_task_originate = 17
- action_task_graphviz = 18
-
-
-
- multiple_jobs_outputs = 0
- single_job_single_output = 1
- job_single_matches_parent= 2
-
- job_limit_semaphores = {}
-
-
-
- #_________________________________________________________________________________________
-
- # create_task / __init__
-
- #_________________________________________________________________________________________
- @staticmethod
- def create_task(func):
- """
- Create task if the name as not been previously specified
- Note that the task function may not have been created yet.
- This allows us to create tasks and dependencies out of order
- """
- func_name = func.__name__
- module_name = str(func.__module__)
- task_name = module_name + "." + func_name
-
- # Link to existing dependency if task name has previously been specified
- if node.is_node(task_name):
- t = node.lookup_node_from_name(task_name)
- if t.user_defined_work_func != None:
- raise error_duplicate_task_name("Same task name %s specified multiple times in the same module" % task_name)
- # otherwise create new
- else:
- t = _task(module_name, func_name)
+# _____________________________________________________________________________
- t.set_action_type (_task.action_task)
- t.user_defined_work_func = func
- assert(t._name == task_name)
- # convert description into one line
- if func.__doc__:
- t._description = re.sub("\n\s+", " ", func.__doc__).strip()
- else:
- t._description = ""
- return t
+def get_name_from_args(named_args):
+ if "name" in named_args:
+ name = named_args["name"]
+ del named_args["name"]
+ return name
+ else:
+ return None
- #_________________________________________________________________________________________
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
- # get_action_name
+# Pipeline
- #_________________________________________________________________________________________
- def get_action_name (self):
- return _task.action_names[self._action_type]
- #_________________________________________________________________________________________
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
- # __init__
+class Pipeline(dict):
+ pipelines = dict()
+ cnt_mkdir = 0
- #_________________________________________________________________________________________
- def __init__ (self, module_name, func_name):
+ def __init__(self, name, *arg, **kw):
"""
- Does nothing because this might just be a dependency.
- If it does not get initialised by a real task
- (a task is depending on an unknown function/task),
- throw an exception when running the pipeline
-
+ Each Ruffus Pipeline object has to have a unique name.
+ "main" is reserved for "main_pipeline", the default pipeline for all
+ Ruffus decorators.
"""
- self._module_name = module_name
- self._func_name = func_name
-
- node.__init__ (self, module_name + "." + func_name)
- self._action_type = _task.action_unspecified
+ # initialise dict
+ super(Pipeline, self).__init__(*arg, **kw)
+
+ # set of tasks
+ self.tasks = set()
+ self.task_names = set()
+
+ # add self to list of all pipelines
+ self.name = name
+ self.original_name = name
+ if name in Pipeline.pipelines:
+ raise Exception("Error:\nDuplicate pipeline. "
+ "A pipeline named '%s' already exists.\n" % name)
+ Pipeline.pipelines[name] = self
+ self.head_tasks = []
+ self.tail_tasks = []
+ self.lookup = dict()
+
+ # _________________________________________________________________________
+
+ # _create_task
+
+ # _________________________________________________________________________
+ def _create_task(self, task_func, task_name=None):
+ """
+ Create task with a function
+ """
+ # add task to main pipeline
+ if not task_name:
+ if task_func.__module__ == "__main__":
+ task_name = task_func.__name__
+ else:
+ task_name = str(task_func.__module__) + \
+ "." + task_func.__name__
+
+ if task_name not in self:
+ task = Task(task_func, task_name, self)
+
+ # task_name already there as the identifying task_name.
+ # If the task_func also matches everything is fine
+ elif (task_name in self.task_names and
+ self[task_name].user_defined_work_func == task_func):
+ task = self[task_name]
+
+ # If the task name is already taken but with a different function,
+ # this will blow up
+ # But if the function is being reused and with a previously different
+ # task name then OK
+ else:
+ task = Task(task_func, task_name, self)
- # Each task has its own checksum level
- # At the moment this is really so multiple pipelines in the same script can have
- # different checksum levels
- # Though set by pipeline_xxxx functions, have initial valid value so unit tests work :-|
- self.checksum_level = CHECKSUM_FILE_TIMESTAMPS
- self.param_generator_func = None
- self.needs_update_func = None
- self.job_wrapper = job_wrapper_generic
+ return task
- #
- self.job_descriptor = generic_job_descriptor
+ # _________________________________________________________________________
- # jobs which produce a single output.
- # special handling for task.get_output_files for dependency chaining
- self._single_job_single_output = self.multiple_jobs_outputs
- self.single_multi_io = self.many_to_many
+ # _complete_task_setup
- # function which is decorated and does the actual work
- self.user_defined_work_func = None
+ # _________________________________________________________________________
+ def _complete_task_setup(self, processed_tasks):
+ """
+ Finishes initialising all tasks
+ Make sure all tasks in dependency list are linked to real functions
+ """
- # functions which will be called when task completes
- self.posttask_functions = []
- # give makedir automatically made parent tasks unique names
- self.cnt_task_mkdir = 0
+ processed_pipelines = set([self.name])
+ unprocessed_tasks = deque(self.tasks)
+ while len(unprocessed_tasks):
+ task = unprocessed_tasks.popleft()
+ if task in processed_tasks:
+ continue
+ processed_tasks.add(task)
+ for ancestral_task in task._complete_setup():
+ if ancestral_task not in processed_tasks:
+ unprocessed_tasks.append(ancestral_task)
+ processed_pipelines.add(ancestral_task.pipeline.name)
+ #
+ # some jobs single state status mirrors parent's state
+ # and parent task not known until dependencies resolved
+ # Is this legacy code?
+ # Breaks @merge otherwise
+ #
+ if isinstance(task._is_single_job_single_output, Task):
+ task._is_single_job_single_output = \
+ task._is_single_job_single_output._is_single_job_single_output
- # whether only task function itself knows what output it will produce
- # i.e. output is a glob or something similar
- self.indeterminate_output = 0
- # cache output file names here
- self.output_filenames = None
+ for pipeline_name in list(processed_pipelines):
+ if pipeline_name != self.name:
+ processed_pipelines |= self.pipelines[pipeline_name]._complete_task_setup(processed_tasks)
- self.semaphore_name = module_name + "." + func_name
+ return processed_pipelines
- # do not test for whether task is active
- self.active_if_checks = None
+ # _________________________________________________________________________
- # extra flag for outputfiles
- self.is_active = True
+ # get_head_tasks
+ # _________________________________________________________________________
+ def get_head_tasks(self):
+ """
+ Return tasks at the head of the pipeline,
+ i.e. with only descendants/dependants
+ N.B. Head and Tail sets can overlap
+ Most of the time when self.head_tasks == [], it has been left undefined by mistake.
+ So we usually throw an exception at the point of use
+ """
+ return self.head_tasks
- #_________________________________________________________________________________________
+ # _________________________________________________________________________
- # init_for_pipeline
+ # set_head_tasks
- #_________________________________________________________________________________________
- def init_for_pipeline (self):
+ # _________________________________________________________________________
+ def set_head_tasks(self, head_tasks):
"""
- Initialize variables for pipeline run / printout
+ Specifies tasks at the head of the pipeline,
+ i.e. with only descendants/dependants
+ """
+ if not isinstance(head_tasks, (list,)):
+ raise Exception("Pipelines['{pipeline_name}'].set_head_tasks() expects a "
+ "list not {head_tasks_type}".format(pipeline_name = self.name,
+ head_tasks_type = type(head_tasks)))
- **********
- BEWARE
- **********
+ for tt in head_tasks:
+ if not isinstance(tt, (Task,)):
+ raise Exception("Pipelines['{pipeline_name}'].set_head_tasks() expects a "
+ "list of tasks not {task_type} {task}".format( pipeline_name = self.name,
+ task_type = type(tt),
+ task = 1))
+ self.head_tasks = head_tasks
- Because state is stored, ruffus is *not* reentrant.
+ # _________________________________________________________________________
- **********
- BEWARE
- **********
- """
+ # get_tail_tasks
- # cache output file names here
- self.output_filenames = None
+ # _________________________________________________________________________
+ def get_tail_tasks(self):
+ """
+ Return tasks at the tail of the pipeline,
+ i.e. without descendants/dependants
+ N.B. Head and Tail sets can overlap
+ Most of the time when self.tail_tasks == [],
+ it has been left undefined by mistake.
+ So we usually throw an exception at the point of use
+ """
+ return self.tail_tasks
- #_________________________________________________________________________________________
+ # _________________________________________________________________________
- # set_action_type
+ # set_tail_tasks
- #_________________________________________________________________________________________
- def set_action_type (self, new_action_type):
+ # _________________________________________________________________________
+ def set_tail_tasks(self, tail_tasks):
"""
- Save how this task
- 1) tests whether it is up-to-date and
- 2) handles input/output files
+ Specifies tasks at the tail of the pipeline,
+ i.e. with only descendants/dependants
+ """
+ self.tail_tasks = tail_tasks
- Checks that the task has not been defined with conflicting actions
+ # _________________________________________________________________________
+
+ # set_input
+ # forward to head tasks
+
+ # _________________________________________________________________________
+ def set_input(self, **args):
"""
- if self._action_type not in (_task.action_unspecified, _task.action_task):
- old_action = _task.action_names[self._action_type]
- new_action = _task.action_names[new_action_type]
- actions = " and ".join(list(set((old_action, new_action))))
- task_name = "def %s(...)" % self._name.replace("__main__.", "")
- raise error_decorator_args((" %s\n has duplicate task specifications: (%s)\n") %
- (task_name, actions))
- self._action_type = new_action_type
- self._action_type_desc = _task.action_names[new_action_type]
+ Change the input parameter(s) of the designated "head" tasks of the pipeline
+ """
+ if not len(self.get_head_tasks()):
+ raise error_no_head_tasks("Pipeline '{pipeline_name}' has no head tasks defined.\n"
+ "Which task in '{pipeline_name}' do you want "
+ "to set_input() for?".format(pipeline_name = self.name))
+ for tt in self.get_head_tasks():
+ tt.set_input(**args)
+ # _________________________________________________________________________
- #_________________________________________________________________________________________
+ # set_output
- # get_job_name
+ # forward to head tasks
- #_________________________________________________________________________________________
- def get_job_name(self, descriptive_param, verbose_abbreviated_path, runtime_data):
+ # _________________________________________________________________________
+ def set_output(self, **args):
"""
- Use job descriptor to return short name for job, including any parameters
+ Change the output parameter(s) of the designated "head" tasks of the pipeline
+ """
+ if not len(self.get_head_tasks()):
+ raise error_no_head_tasks("Pipeline '{pipeline_name}' has no head tasks defined.\n"
+ "Which task in '{pipeline_name}' do you want "
+ "to set_output() for?".format(pipeline_name = self.name))
+
+ for tt in self.get_head_tasks():
+ tt.set_output(**args)
+ # _________________________________________________________________________
- runtime_data is not (yet) used but may be used to add context in future
+ # clone
+
+ # _________________________________________________________________________
+ def clone(self, new_name, *arg, **kw):
+ """
+ Make a deep copy of the pipeline
"""
- return self.job_descriptor(descriptive_param, verbose_abbreviated_path, runtime_data)[0]
+ # setup new pipeline
+ new_pipeline = Pipeline(new_name, *arg, **kw)
- #_________________________________________________________________________________________
+ # set of tasks
+ new_pipeline.tasks = set(task._clone(new_pipeline) for task in self.tasks)
+ new_pipeline.task_names = set(self.task_names)
- # get_task_name
+ # so keep original name after a series of cloning operations
+ new_pipeline.original_name = self.original_name
- #_________________________________________________________________________________________
- def get_task_name(self, in_func_format = False):
- """
- Returns name of task function, removing __main__ namespace if necessary
+ # lookup tasks in new pipeline
+ new_pipeline.head_tasks = [new_pipeline[t._name] for t in self.head_tasks]
+ new_pipeline.tail_tasks = [new_pipeline[t._name] for t in self.tail_tasks]
+
+ return new_pipeline
- if in_func_format is true, will return def task_func(...):
+ # _________________________________________________________________________
+ # mkdir
+
+ # _________________________________________________________________________
+ def mkdir(self, *unnamed_args, **named_args):
+ """
+ Makes directories each incoming input to a corresponding output
+ This is a One to One operation
"""
+ name = get_name_from_args(named_args)
+ # func is a placeholder...
+ if name is None:
+ self.cnt_mkdir += 1
+ if self.cnt_mkdir == 1:
+ name = "mkdir"
+ else:
+ name = "mkdir # %d" % self.cnt_mkdir
+ task = self._create_task(task_func=job_wrapper_mkdir, task_name=name)
+ task.created_via_decorator = False
+ task.syntax = "pipeline.mkdir"
+ task.description_with_args_placeholder = "%s(name = %r, %%s)" % (
+ task.syntax, task._get_display_name())
+ task._prepare_mkdir(unnamed_args, named_args, task.description_with_args_placeholder)
+ return task
- task_name = self._name.replace("__main__.", "")
- if self._action_type != _task.action_mkdir and in_func_format:
- return "def %s(...):" % task_name
- else:
- return task_name
+ # _________________________________________________________________________
+ # _do_create_task_by_OOP
+ # _________________________________________________________________________
+ def _do_create_task_by_OOP(self, task_func, named_args, syntax):
+ """
+ Helper function for
+ Pipeline.transform
+ Pipeline.originate
+ pipeline.split
+ pipeline.subdivide
+ pipeline.parallel
+ pipeline.files
+ pipeline.combinations_with_replacement
+ pipeline.combinations
+ pipeline.permutations
+ pipeline.product
+ pipeline.collate
+ pipeline.merge
+ """
+ name = get_name_from_args(named_args)
+ task = self._create_task(task_func, name)
+ task.created_via_decorator = False
+ task.syntax = syntax
+ task.description_with_args_placeholder = "{syntax}(name = {task_display_name!r}, task_func = {task_func_name}, %s)" \
+ .format(syntax = task.syntax,
+ task_display_name = task._get_display_name(),
+ task_func_name = task_func.__name__)
+ return task
+
+ # _________________________________________________________________________
+
+ # lookup_task_from_name
+
+ # _________________________________________________________________________
+ def lookup_task_from_name(self, task_name, default_module_name):
+ """
+ If lookup returns None, means ambiguous: do nothing
+ Only ever returns a list of one
+ """
+ multiple_tasks = []
- #_________________________________________________________________________________________
+ #
+ # Does the unqualified name uniquely identify?
+ #
+ if task_name in self.lookup:
+ if len(self.lookup[task_name]) == 1:
+ return self.lookup[task_name]
+ else:
+ multiple_tasks = self.lookup[task_name]
- # update_active_state
+ #
+ # Even if the unqualified name does not uniquely identify,
+ # maybe the qualified name does
+ #
+ full_qualified_name = default_module_name + "." + task_name
+ if full_qualified_name in self.lookup:
+ if len(self.lookup[full_qualified_name]) == 1:
+ return self.lookup[full_qualified_name]
+ else:
+ multiple_tasks = self.lookup[task_name]
- #_________________________________________________________________________________________
- def update_active_state (self):
#
- # If has an @active_if decorator, check if the task needs to be run
- # @active_if parameters may be call back functions or booleans
+ # Nothing matched
#
- if (self.active_if_checks != None and
- any( not arg() if isinstance(arg, collections.Callable) else not arg
- for arg in self.active_if_checks)):
- # flip is active to false.
- # ( get_output_files() will return empty if inactive )
- # Remember each iteration of pipeline_printout pipeline_run will have
- # another bite at changing this value
- self.is_active = False
- else:
- # flip is active to True so that downstream dependencies will be correct
- # ( get_output_files() will return empty if inactive )
- # Remember each iteration of pipeline_printout pipeline_run will have
- # another bite at changing this value
- self.is_active = True
+ if not multiple_tasks:
+ return []
+
+ #
+ # If either the qualified or unqualified name is ambiguous, throw...
+ #
+ task_names = ",".join(t._name for t in multiple_tasks)
+ raise error_ambiguous_task("%s is ambiguous. Which do you mean? (%s)."
+ % (task_name, task_names))
+ # _________________________________________________________________________
+ # follows
- #_________________________________________________________________________________________
+ # _________________________________________________________________________
+ def follows(self, task_func, *unnamed_args, **named_args):
+ """
+ Transforms each incoming input to a corresponding output
+ This is a One to One operation
+ """
+ task = self._do_create_task_by_OOP(task_func, named_args, "pipeline.follows")
+ task.deferred_follow_params.append([task.description_with_args_placeholder, False,
+ unnamed_args])
+ #task._connect_parents(task.description_with_args_placeholder, False,
+ # unnamed_args)
+ return task
+ # _________________________________________________________________________
+
+ # check_if_uptodate
+
+ # _________________________________________________________________________
+ def check_if_uptodate(self, task_func, func, **named_args):
+ """
+ Specifies how a task is to be checked if it needs to be rerun (i.e. is
+ up-to-date).
+ func returns true if input / output files are up to date
+ func takes as many arguments as the task function
+ """
+ task = self._do_create_task_by_OOP(task_func, named_args, "check_if_uptodate")
+ return task.check_if_uptodate(func)
- # printout
+ # _________________________________________________________________________
- # This code will look so much better once we have job level dependencies
- # pipeline_run has dependencies percolating up/down. Don't want to
- # recreate all the logic here
+ # graphviz
- #_________________________________________________________________________________________
- def printout (self, runtime_data, force_rerun, job_history, task_is_out_of_date, verbose=1, verbose_abbreviated_path = 2, indent = 4):
+ # _________________________________________________________________________
+ def graphviz(self, task_func, *unnamed_args, **named_args):
"""
- Print out all jobs for this task
+ Transforms each incoming input to a corresponding output
+ This is a One to One operation
+ """
+ task = self._do_create_task_by_OOP(task_func, named_args, "pipeline.graphviz")
+ task.graphviz_attributes = named_args
+ if len(unnamed_args):
+ raise TypeError("Only named arguments expected in :" +
+ task.description_with_args_placeholder % unnamed_args)
+ return task
+ # _________________________________________________________________________
+
+ # transform
+
+ # _________________________________________________________________________
+ def transform(self, task_func, *unnamed_args, **named_args):
+ """
+ Transforms each incoming input to a corresponding output
+ This is a One to One operation
+ """
+ task = self._do_create_task_by_OOP(task_func, named_args, "pipeline.transform")
+ task._prepare_transform(unnamed_args, named_args)
+ return task
- verbose =
- level 1 : logs Out-of-date Task names
- level 2 : logs All Tasks (including any task function docstrings)
- level 3 : logs Out-of-date Jobs in Out-of-date Tasks, no explanation
- level 4 : logs Out-of-date Jobs in Out-of-date Tasks, saying why they are out of date (include only list of up-to-date tasks)
- level 5 : All Jobs in Out-of-date Tasks (include only list of up-to-date tasks)
- level 6 : All jobs in All Tasks whether out of date or not
+ # _________________________________________________________________________
+ # originate
+
+ # _________________________________________________________________________
+ def originate(self, task_func, *unnamed_args, **named_args):
+ """
+ Originates a new set of output files,
+ one output per call to the task function
"""
+ task = self._do_create_task_by_OOP(task_func, named_args, "pipeline.originate")
+ task._prepare_originate(unnamed_args, named_args)
+ return task
- def get_job_names (param, indent_str):
- job_names = self.job_descriptor(param, verbose_abbreviated_path, runtime_data)[1]
- if len(job_names) > 1:
- job_names = ([indent_str + job_names[0]] +
- [indent_str + " " + jn for jn in job_names[1:]])
- else:
- job_names = ([indent_str + job_names[0]])
- return job_names
+ # _________________________________________________________________________
+ # split
+ # _________________________________________________________________________
+ def split(self, task_func, *unnamed_args, **named_args):
+ """
+ Splits a single set of input files into multiple output file names,
+ where the number of output files may not be known beforehand.
+ This is a One to Many operation
+ """
+ task = self._do_create_task_by_OOP(task_func, named_args, "pipeline.split")
+ task._prepare_split(unnamed_args, named_args)
+ return task
- if not verbose:
- return []
+ # _________________________________________________________________________
- indent_str = ' ' * indent
+ # subdivide
- messages = []
+ # _________________________________________________________________________
+ def subdivide(self, task_func, *unnamed_args, **named_args):
+ """
+ Subdivides a each set of input files into multiple output file names,
+ where the number of output files may not be known beforehand.
+ This is a Many to Even More operation
+ """
+ task = self._do_create_task_by_OOP(task_func, named_args, "pipeline.subdivide")
+ task._prepare_subdivide(unnamed_args, named_args)
+ return task
- # LOGGER: level 1 : logs Out-of-date Tasks (names and warnings)
- messages.append("Task = " + self.get_task_name() + (" >>Forced to rerun<<" if force_rerun else ""))
- if verbose ==1:
- return messages
+ # _________________________________________________________________________
- # LOGGER: level 2 : logs All Tasks (including any task function docstrings)
- if verbose >= 2 and len(self._description):
- messages.append(indent_str + '"' + self._description + '"')
+ # merge
- #
- # single job state
- #
- if verbose >= 10:
- if self._single_job_single_output == self.single_job_single_output:
- messages.append(" Single job single output")
- elif self._single_job_single_output == self.multiple_jobs_outputs:
- messages.append(" Multiple jobs Multiple outputs")
- else:
- messages.append(" Single jobs status depends on %s" % self._single_job_single_output._name)
+ # _________________________________________________________________________
+ def merge(self, task_func, *unnamed_args, **named_args):
+ """
+ Merges multiple input files into a single output.
+ This is a Many to One operation
+ """
+ task = self._do_create_task_by_OOP(task_func, named_args, "pipeline.merge")
+ task._prepare_merge(unnamed_args, named_args)
+ return task
+ # _________________________________________________________________________
- # LOGGER: No job if less than 2
- if verbose <= 2 :
- return messages
+ # collate
- # increase indent for jobs up to date status
- indent_str += " " * 3
+ # _________________________________________________________________________
+ def collate(self, task_func, *unnamed_args, **named_args):
+ """
+ Collates each set of multiple matching input files into an output.
+ This is a Many to Fewer operation
+ """
+ task = self._do_create_task_by_OOP(task_func, named_args, "pipeline.collate")
+ task._prepare_collate(unnamed_args, named_args)
+ return task
- #
- # If has an @active_if decorator, check if the task needs to be run
- # @active_if parameters may be call back functions or booleans
- #
- if not self.is_active:
- # LOGGER
- if verbose <= 3:
- return messages
- messages.append(indent_str + "Task is inactive")
- # add spacer line
- messages.append("")
- return messages
+ # _________________________________________________________________________
+ # product
+
+ # _________________________________________________________________________
+ def product(self, task_func, *unnamed_args, **named_args):
+ """
+ All-vs-all Product between items from each set of inputs
+ """
+ task = self._do_create_task_by_OOP(task_func, named_args, "pipeline.product")
+ task._prepare_product(unnamed_args, named_args)
+ return task
+
+ # _________________________________________________________________________
+
+ # permutations
+
+ # _________________________________________________________________________
+ def permutations(self, task_func, *unnamed_args, **named_args):
+ """
+ Permutations between items from a set of inputs
+ * k-length tuples
+ * all possible orderings
+ * no self vs self
+ """
+ task = self._do_create_task_by_OOP(task_func, named_args, "pipeline.permutations")
+ task._prepare_combinatorics(
+ unnamed_args, named_args, error_task_permutations)
+ return task
+
+ # _________________________________________________________________________
+
+ # combinations
+
+ # _________________________________________________________________________
+ def combinations(self, task_func, *unnamed_args, **named_args):
+ """
+ Combinations of items from a set of inputs
+ * k-length tuples
+ * Single (sorted) ordering, i.e. AB is the same as BA,
+ * No repeats. No AA, BB
+ For Example:
+ combinations("ABCD", 3) = ['ABC', 'ABD', 'ACD', 'BCD']
+ combinations("ABCD", 2) = ['AB', 'AC', 'AD', 'BC', 'BD', 'CD']
+ """
+ task = self._do_create_task_by_OOP(task_func, named_args, "pipeline.combinations")
+ task._prepare_combinatorics(unnamed_args, named_args, error_task_combinations)
+ return task
+
+ # _________________________________________________________________________
+
+ # combinations_with_replacement
+
+ # _________________________________________________________________________
+ def combinations_with_replacement(self, task_func, *unnamed_args,
+ **named_args):
+ """
+ Combinations with replacement of items from a set of inputs
+ * k-length tuples
+ * Single (sorted) ordering, i.e. AB is the same as BA,
+ * Repeats. AA, BB, AAC etc.
+ For Example:
+ combinations_with_replacement("ABCD", 2) = [
+ 'AA', 'AB', 'AC', 'AD',
+ 'BB', 'BC', 'BD',
+ 'CC', 'CD',
+ 'DD']
+ combinations_with_replacement("ABCD", 3) = [
+ 'AAA', 'AAB', 'AAC', 'AAD',
+ 'ABB', 'ABC', 'ABD',
+ 'ACC', 'ACD',
+ 'ADD',
+ 'BBB', 'BBC', 'BBD',
+ 'BCC', 'BCD',
+ 'BDD',
+ 'CCC', 'CCD',
+ 'CDD',
+ 'DDD']
+ """
+ task = self._do_create_task_by_OOP(task_func, named_args, "combinations_with_replacement")
+ task._prepare_combinatorics(unnamed_args, named_args,
+ error_task_combinations_with_replacement)
+ return task
+
+ # _________________________________________________________________________
+
+ # files
+
+ # _________________________________________________________________________
+ def files(self, task_func, *unnamed_args, **named_args):
+ """
+ calls user function in parallel
+ with either each of a list of parameters
+ or using parameters generated by a custom function
+
+ In the parameter list,
+ The first two items of each set of parameters must
+ be input/output files or lists of files or Null
+ """
+ task = self._do_create_task_by_OOP(task_func, named_args, "pipeline.files")
+ task._prepare_files(unnamed_args, named_args)
+ return task
+
+ # _________________________________________________________________________
+
+ # parallel
+
+ # _________________________________________________________________________
+ def parallel(self, task_func, *unnamed_args, **named_args):
+ """
+ calls user function in parallel
+ with either each of a list of parameters
+ or using parameters generated by a custom function
+ """
+ task = self._do_create_task_by_OOP(task_func, named_args, "pipeline.parallel")
+ task._prepare_parallel(unnamed_args, named_args)
+ return task
+
+ # _________________________________________________________________________
+
+ # run
+ # printout
+ #
+ # Forwarding functions
+ # Should bring procedural function here and forward from the other
+ # direction?
+
+ # _________________________________________________________________________
+ def run(self, *unnamed_args, **named_args):
+ pipeline_run(pipeline=self, *unnamed_args, **named_args)
+
+ def printout(self, *unnamed_args, **named_args):
+ pipeline_printout(pipeline=self, *unnamed_args, **named_args)
+
+ def get_task_names(self, *unnamed_args, **named_args):
+ pipeline_get_task_names(pipeline=self, *unnamed_args, **named_args)
+
+ def printout_graph(self, *unnamed_args, **named_args):
+ pipeline_printout_graph(pipeline=self, *unnamed_args, **named_args)
+
+#
+# Global default shared pipeline (used for decorators)
+#
+main_pipeline = Pipeline(name="main")
+
+
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# Functions
+
+
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
+# _____________________________________________________________________________
+
+# lookup_unique_task_from_func
+
+# _____________________________________________________________________________
+def lookup_unique_task_from_func(task_func, default_pipeline_name="main"):
+ """
+ Go through all pipelines and match task_func to find a unique task
+ Throw exception if ambiguous
+ """
+
+ def unique_task_from_func_in_pipeline(task_func, pipeline):
+ if task_func in pipeline.lookup:
+ if len(pipeline.lookup[task_func]) == 1:
+ # Found task!
+ return pipeline.lookup[task_func][0]
+
+ # Found too many tasks! Ambiguous...
+ task_names = ", ".join(task._name for task in pipeline.lookup[task_func])
+ raise error_ambiguous_task(
+ "Function def %s(...): is used by multiple tasks (%s). Which one do you mean?."
+ % (task_func.__name__, task_names))
+ return None
+
+ #
+ # Iterate through all pipelines starting with the specified pipeline
+ #
+ task = unique_task_from_func_in_pipeline(task_func, Pipeline.pipelines[default_pipeline_name])
+ if task:
+ return task
+
+ #
+ # Sees if function uniquely identifies a single task across pipelines
+ #
+ found_tasks = []
+ found_pipelines = []
+ for pipeline in Pipeline.pipelines.values():
+ task = unique_task_from_func_in_pipeline(task_func, pipeline)
+ if task:
+ found_tasks.append(task)
+ found_pipelines.append(pipeline)
+
+ if len(found_tasks) == 1:
+ return found_tasks[0]
+
+ if len(found_tasks) > 1:
+ raise error_ambiguous_task("Task Name %s is ambiguous and specifies different tasks "
+ "across multiple pipelines (%s)."
+ % (task_func.__name__, ",".join(found_pipelines)))
+
+ return None
+
+
+# _____________________________________________________________________________
+
+# lookup_tasks_from_name
+
+# _____________________________________________________________________________
+def lookup_tasks_from_name(task_name, default_pipeline_name, default_module_name="__main__"):
+ """
+
+ Tries:
+ (1) Named pipeline in the format pipeline::task_name
+ (2) tasks matching task_name in default_pipeline_name
+ (3) pipeline names matching task_name
+ (4) if task_name uniquely identifies any task in all other pipelines...
+
+ Only returns multiple tasks if (3) task_name is the name of a pipeline
+ """
+
+ # Lookup the task from the function or task name
+ pipeline_name, task_name = re.match("(?:(.+)::)?(.*)", task_name).groups()
+
+ #
+ # (1) Look in specified pipeline
+ # Will blow up if task_name is ambiguous
+ #
+ if pipeline_name:
+ if pipeline_name not in Pipeline.pipelines:
+ raise error_not_a_pipeline("%s is not a pipeline." % pipeline_name)
+ pipeline = Pipeline.pipelines[pipeline_name]
+ return pipeline.lookup_task_from_name(task_name, default_module_name)
+
+ #
+ # (2) Try default pipeline
+ # Will blow up if task_name is ambiguous
+ #
+ if default_pipeline_name not in Pipeline.pipelines:
+ raise error_not_a_pipeline("%s is not a pipeline." % default_pipeline_name)
+ pipeline = Pipeline.pipelines[default_pipeline_name]
+ tasks = pipeline.lookup_task_from_name(task_name, default_module_name)
+ if tasks:
+ return tasks
+
+ # (3) task_name is actually the name of a pipeline
+ # Alias for pipeline.get_tail_tasks()
+ # N.B. This is the *only* time multiple tasks might be returned
+ #
+ if task_name in Pipeline.pipelines:
+ if not len(Pipeline.pipelines[task_name].get_tail_tasks()):
+ raise error_no_tail_tasks(
+ "Pipeline %s has no tail tasks defined. Which task do you "
+ "mean when you specify the whole pipeline as a dependency?" % task_name)
+ return Pipeline.pipelines[task_name].get_tail_tasks()
+
+ #
+ # (4) Try all other pipelines
+ # Will blow up if task_name is ambiguous
+ #
+ found_tasks = []
+ found_pipelines = []
+ for pipeline_name, pipeline in Pipeline.pipelines.items():
+ tasks = pipeline.lookup_task_from_name(task_name, default_module_name)
+ if tasks:
+ found_tasks.append(tasks)
+ found_pipelines.append(pipeline_name)
+
+ # unambiguous: good
+ if len(found_tasks) == 1:
+ return found_tasks[0]
+
+ # ambiguous: bad
+ if len(found_tasks) > 1:
+ raise error_ambiguous_task(
+ "Task Name %s is ambiguous and specifies different tasks across "
+ "several pipelines (%s)." % (task_name, ",".join(found_pipelines)))
+
+ # Nothing found
+ return []
+
+
+# _____________________________________________________________________________
+
+# lookup_tasks_from_user_specified_names
+#
+# _____________________________________________________________________________
+def lookup_tasks_from_user_specified_names(task_description, task_names,
+ default_pipeline_name="main",
+ default_module_name="__main__"):
+ """
+ Given a list of task names, look up the corresponding tasks
+ Will just pass through if the task_name is already a task
+ """
+
+ #
+ # In case we are given a single item instead of a list
+ #
+ if not isinstance(task_names, (list, tuple)):
+ task_names = [task_names]
+
+ task_list = []
+
+ for task_name in task_names:
+
+ # "task_name" is a Task or pipeline, add those
+ if isinstance(task_name, Task):
+ task_list.append(task_name)
+ continue
+
+ elif isinstance(task_name, Pipeline):
+ if not len(task_name.get_tail_tasks()):
+ raise error_no_tail_tasks("Pipeline %s has no 'tail tasks'. Which task do you mean"
+ " when you specify the whole pipeline?" % task_name.name)
+ task_list.extend(task_name.get_tail_tasks())
+ continue
+
+ if isinstance(task_name, collections.Callable):
+ # blows up if ambiguous
+ task = lookup_unique_task_from_func(task_name, default_pipeline_name)
+ # blow up for unwrapped function
+ if not task:
+ raise error_function_is_not_a_task(
+ ("Function def %s(...): is not a Ruffus task." % task_func.__name__) +
+ " The function needs to have a ruffus decoration like "
+ "'@transform', or be a member of a ruffus.Pipeline().")
+
+ task_list.append(task)
+ continue
+
+ # some kind of string: task or func or pipeline name?
+ if isinstance(task_name, path_str_type):
+
+ # Will throw Exception if ambiguous
+ tasks = lookup_tasks_from_name(
+ task_name, default_pipeline_name, default_module_name)
+ # not found
+ if not tasks:
+ raise error_node_not_task("%s task '%s' is not a pipelined task in Ruffus. Is it "
+ "spelt correctly ?" % (task_description, task_name))
+ task_list.extend(tasks)
+ continue
+
+ else:
+ raise TypeError("Expecting a string or function, or a Ruffus Task or Pipeline object")
+ return task_list
+
+
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# Task
+
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
+class Task (node):
+
+ """
+
+ * Represents each stage of a pipeline.
+ * Associated with a single python function.
+ * Identified uniquely within the pipeline by its name.
+
+ """
+
+ #DEBUGGG
+ #def __str__ (self):
+ # return "Task = <%s>" % self._get_display_name()
+
+ _action_names = ["unspecified",
+ "task",
+ "task_files_re",
+ "task_split",
+ "task_merge",
+ "task_transform",
+ "task_collate",
+ "task_files_func",
+ "task_files",
+ "task_mkdir",
+ "task_parallel",
+ "task_active_if",
+ "task_product",
+ "task_permutations",
+ "task_combinations",
+ "task_combinations_with_replacement",
+ "task_subdivide",
+ "task_originate",
+ "task_graphviz",
+ ]
+ # ENUMS
+ (_action_unspecified,
+ _action_task,
+ _action_task_files_re,
+ _action_task_split,
+ _action_task_merge,
+ _action_task_transform,
+ _action_task_collate,
+ _action_task_files_func,
+ _action_task_files,
+ _action_mkdir,
+ _action_task_parallel,
+ _action_active_if,
+ _action_task_product,
+ _action_task_permutations,
+ _action_task_combinations,
+ _action_task_combinations_with_replacement,
+ _action_task_subdivide,
+ _action_task_originate,
+ _action_task_graphviz) = range(19)
+
+ (_multiple_jobs_outputs,
+ _single_job_single_output,
+ _job_single_matches_parent) = range(3)
+
+ _job_limit_semaphores = {}
+
+ # _________________________________________________________________________
+
+ # _get_action_name
+
+ # _________________________________________________________________________
+ def _get_action_name(self):
+ return Task._action_names[self._action_type]
+
+ # _________________________________________________________________________
+
+ # __init__
+
+ # _________________________________________________________________________
+ def __init__(self, func, task_name=None, pipeline=None):
+ """
+ * Creates a Task object with a specified python function and task name
+ * The type of the Task (whether it is a transform or merge or collate
+ etc. operation) is specified subsequently. This is because Ruffus
+ decorators do not have to be specified in order, and we don't
+ know ahead of time.
+ """
+ if pipeline is None:
+ pipeline = main_pipeline
+ self.pipeline = pipeline
+ self.func_module_name = str(func.__module__)
+ self.func_name = func.__name__
+ # convert description into one line
+ self.func_description = re.sub("\n\s+", " ", func.__doc__).strip() if func.__doc__ else ""
+
+ if not task_name:
+ task_name = self.func_module_name + "." + self.func_name
+
+
+ node.__init__(self, task_name)
+ self._action_type = Task._action_task
+ self._action_type_desc = Task._action_names[self._action_type]
+
+ # Each task has its own checksum level
+ # At the moment this is really so multiple pipelines in the same
+ # script can have different checksum levels
+ # Though set by pipeline_xxxx functions, have initial valid value so
+ # unit tests work :-|
+ self.checksum_level = CHECKSUM_FILE_TIMESTAMPS
+ self.param_generator_func = None
+ self.needs_update_func = None
+ self.job_wrapper = job_wrapper_generic
+
+ #
+ self.job_descriptor = generic_job_descriptor
+
+ # jobs which produce a single output.
+ # special handling for task.get_output_files for dependency chaining
+ self._is_single_job_single_output = self._multiple_jobs_outputs
+ self.single_multi_io = self._many_to_many
+
+ # function which is decorated and does the actual work
+ self.user_defined_work_func = func
+
+ # functions which will be called when task completes
+ self.posttask_functions = []
+
+ # give makedir automatically made parent tasks unique names
+ self.cnt_task_mkdir = 0
+
+ # whether only task function itself knows what output it will produce
+ # i.e. output is a glob or something similar
+ self.indeterminate_output = 0
+
+ # cache output file names here
+ self.output_filenames = None
+
+ # semaphore name must be unique
+ self.semaphore_name = pipeline.name + ":" + task_name
+
+ # do not test for whether task is active
+ self.active_if_checks = None
+
+ # extra flag for outputfiles
+ self.is_active = True
+
+ # Created via decorator or OO interface
+ # so that display_name looks more natural
+ self.created_via_decorator = False
+
+ # Finish setting up task
+ self._setup_task_func = Task._do_nothing_setup
+
+ # Finish setting up task
+ self.deferred_follow_params = []
+
+ # Finish setting up task
+ self.parsed_args = {}
+ self.error_type = None
+
+ # @split or pipeline.split etc.
+ self.syntax = ""
+
+ self.description_with_args_placeholder = "%s"
+
+ # whether task has a (re-specifiable) input parameter
+ self.has_input_param = True
+ self.has_pipeline_in_input_param = False
+
+ # add to pipeline's lookup
+ # this code is here rather than the pipeline so that current unittests
+ # do not need to bother about pipeline
+ if task_name in self.pipeline.task_names:
+ raise error_duplicate_task_name("Same task name %s specified multiple times in the "
+ "same pipeline (%s)" % (task_name, self.pipeline.name))
+
+ self.pipeline.tasks.add(self)
+
+ # task_name is always a unique lookup and overrides everything else
+ self.pipeline[task_name] = self
+ self.pipeline.lookup[task_name] = [self]
+ self.pipeline.task_names.add(task_name)
+
+ #
+ # Allow pipeline to lookup task by
+ # 1) Func
+ # 2) task name
+ # 3) func name
+ #
+ # Ambiguous func names returns an empty list []
+ #
+
+ for lookup in (func, self.func_name, self.func_module_name + "." + self.func_name):
+ # don't add to lookup if this conflicts with a task_name which is
+ # always unique and overriding
+ if lookup not in self.pipeline.task_names:
+ # non-unique map
+ if lookup in self.pipeline.lookup:
+ self.pipeline.lookup[lookup].append(self)
+ # remove non-uniques from Pipeline
+ if lookup in self.pipeline:
+ del self.pipeline[lookup]
+ else:
+ self.pipeline.lookup[lookup] = [self]
+ self.pipeline[lookup] = self
+
+ #
+
+ # _________________________________________________________________________
+
+ # _clone
+
+ # _________________________________________________________________________
+ def _clone(self, pipeline):
+ """
+ * Clones a Task object from self
+ """
+ new_task = Task(self.user_defined_work_func, self._name, pipeline)
+ new_task._action_type = self._action_type
+ new_task._action_type_desc = self._action_type_desc
+ new_task.checksum_level = self.checksum_level
+ new_task.param_generator_func = self.param_generator_func
+ new_task.needs_update_func = self.needs_update_func
+ new_task.job_wrapper = self.job_wrapper
+ new_task.job_descriptor = self.job_descriptor
+ new_task._is_single_job_single_output = self._is_single_job_single_output
+ new_task.single_multi_io = self.single_multi_io
+ new_task.posttask_functions = copy.deepcopy(self.posttask_functions)
+ new_task.cnt_task_mkdir = self.cnt_task_mkdir
+ new_task.indeterminate_output = self.indeterminate_output
+ new_task.semaphore_name = self.semaphore_name
+ new_task.is_active = self.is_active
+ new_task.created_via_decorator = self.created_via_decorator
+ new_task._setup_task_func = self._setup_task_func
+ new_task.error_type = self.error_type
+ new_task.syntax = self.syntax
+ new_task.description_with_args_placeholder = self.description_with_args_placeholder
+ new_task.has_input_param = self.has_input_param
+ new_task.has_pipeline_in_input_param = self.has_pipeline_in_input_param
+ new_task.output_filenames = copy.deepcopy(self.output_filenames)
+ new_task.active_if_checks = copy.deepcopy(self.active_if_checks)
+ new_task.parsed_args = copy.deepcopy(self.parsed_args)
+ new_task.deferred_follow_params = copy.deepcopy(self.deferred_follow_params)
+
+ return new_task
+
+
+ # _________________________________________________________________________
+
+ # set_output
+
+ # _________________________________________________________________________
+ def set_output(self, **args):
+ """
+ Changes output parameter(s) for originate
+ set_input(output = "test.txt")
+ """
+
+ if self.syntax not in ("pipeline.originate", "@originate"):
+ raise error_set_output("Can only set output for originate tasks")
+ #
+ # For product: filter parameter is a list of formatter()
+ #
+ if "output" in args:
+ self.parsed_args["output"] = args["output"]
+ del args["output"]
+ else:
+ raise error_set_output("Missing the output argument in set_input(output=xxx)")
+
+ # Non "input" arguments
+ if len(args):
+ raise error_set_output("Unexpected argument name in set_output(%s). "
+ "Only expecting output=xxx." % (args,))
+ # _________________________________________________________________________
+
+ # set_input
+
+ # _________________________________________________________________________
+ def set_input(self, **args):
+ """
+ Changes any of the input parameter(s) of the task
+ For example:
+ set_input(input = "test.txt")
+ set_input(input2 = "b.txt")
+ set_input(input = "a.txt", input2 = "b.txt")
+ """
+ #
+ # For product: filter parameter is a list of formatter()
#
- # No parameters: just call task function
- #
- if self.param_generator_func == None:
+ if ("filter" in self.parsed_args and
+ isinstance(self.parsed_args["filter"], list)):
+ # the number of input is the count of filter
+ cnt_expected_input = len(self.parsed_args["filter"])
+
+ # make sure the parsed parameter argument is setup, with empty
+ # lists if necessary
+ # Should have been done already...
+ # if self.parsed_args["input"] is None:
+ # self.parsed_args["input"] = [[]
+ # for i in range(cnt_expected_input)]
+
+ # update each element of the list accordingly
+ # removing args so we can check if there is anything left over
+ for inputN in range(cnt_expected_input):
+ input_name = "input%d" % (inputN + 1) if inputN else "input"
+ if input_name in args:
+ self.parsed_args["input"][inputN] = args[input_name]
+ del args[input_name]
+
+ if len(args):
+ raise error_set_input("Unexpected arguments in set_input(%s). "
+ "Only expecting inputN=xxx" % (args,))
+ return
+
+ if "input" in args:
+ self.parsed_args["input"] = args["input"]
+ del args["input"]
+ else:
+ raise error_set_input("Missing the input argument in set_input(input=xxx)")
+
+ # Non "input" arguments
+ if len(args):
+ raise error_set_input("Unexpected argument name in set_input(%s). "
+ "Only expecting input=xxx." % (args,))
+
+ # _________________________________________________________________________
+
+ # _init_for_pipeline
+
+ # _________________________________________________________________________
+ def _init_for_pipeline(self):
+ """
+ Initialize variables for pipeline run / printout
+
+ **********
+ BEWARE
+ **********
+
+ Because state is stored, ruffus is *not* reentrant.
+
+ TODO: Need to create runtime DAG to mirror task DAG which holds
+ output_filenames to be reentrant
+
+ **********
+ BEWARE
+ **********
+ """
+
+ # cache output file names here
+ self.output_filenames = None
+
+ # _________________________________________________________________________
+
+ # _set_action_type
+
+ # _________________________________________________________________________
+ def _set_action_type(self, new_action_type):
+ """
+ Save how this task
+ 1) tests whether it is up-to-date and
+ 2) handles input/output files
+
+ Checks that the task has not been defined with conflicting actions
+
+ """
+ if self._action_type not in (Task._action_unspecified, Task._action_task):
+ old_action = Task._action_names[self._action_type]
+ new_action = Task._action_names[new_action_type]
+ actions = " and ".join(list(set((old_action, new_action))))
+ raise error_decorator_args("%s\n has conflicting task specifications: (%s)\n" %
+ (self.description_with_args_placeholder % "...", actions))
+ self._action_type = new_action_type
+ self._action_type_desc = Task._action_names[new_action_type]
+
+ # _________________________________________________________________________
+
+ # _get_job_name
+
+ # _________________________________________________________________________
+ def _get_job_name(self, descriptive_param, verbose_abbreviated_path, runtime_data):
+ """
+ Use job descriptor to return short name for job including any parameters
+
+ runtime_data is not (yet) used but may be used to add context in future
+ """
+ return self.job_descriptor(descriptive_param, verbose_abbreviated_path, runtime_data)[0]
+
+ # _________________________________________________________________________
+
+ # _get_display_name
+
+ # _________________________________________________________________________
+ def _get_display_name(self):
+ """
+ Returns task name, removing __main__. namespace or main. if present
+ """
+ if self.pipeline.name != "main":
+ return "{pipeline_name}.{task_name}".format(pipeline_name = self.pipeline.name,
+ task_name = self._name.replace("__main__.", "").replace("main.", ""))
+ else:
+ return self._name.replace("__main__.", "").replace("main.", "")
+
+ # _________________________________________________________________________
+
+ # _get_decorated_function
+
+ # _________________________________________________________________________
+ def _get_decorated_function(self):
+ """
+ Returns name of task function, removing __main__ namespace if necessary
+ If not specified using decorator notation, returns empty string
+ N.B. Returns trailing new line
+
+ """
+ if not self.created_via_decorator:
+ return ""
+
+ func_name = (self.func_module_name + "." +
+ self.func_name) \
+ if self.func_module_name != "__main__" else self.func_name
+ return "def %s(...):\n ...\n" % func_name
+
+ # _________________________________________________________________________
+
+ # _update_active_state
+
+ # _________________________________________________________________________
+ def _update_active_state(self):
+ #
+ # If has an @active_if decorator, check if the task needs to be run
+ # @active_if parameters may be call back functions or booleans
+ #
+ if (self.active_if_checks is not None and
+ any(not arg() if isinstance(arg, collections.Callable) else not arg
+ for arg in self.active_if_checks)):
+ # flip is active to false.
+ # ( get_output_files() will return empty if inactive )
+ # Remember each iteration of pipeline_printout pipeline_run
+ # will have another bite at changing this value
+ self.is_active = False
+ else:
+ # flip is active to True so that downstream dependencies will be
+ # correct ( get_output_files() will return empty if inactive )
+ # Remember each iteration of pipeline_printout pipeline_run will
+ # have another bite at changing this value
+ self.is_active = True
+
+ # _________________________________________________________________________
+
+ # _printout
+
+ # This code will look much better once we have job level dependencies
+ # pipeline_run has dependencies percolating up/down. Don't want
+ # to recreate all the logic here
+
+ # _________________________________________________________________________
+ def _printout(self, runtime_data, force_rerun, job_history, task_is_out_of_date, verbose=1,
+ verbose_abbreviated_path=2, indent=4):
+ """
+ Print out all jobs for this task
+
+ verbose =
+ level 1 : logs Out-of-date Task names
+ level 2 : logs All Tasks (including any task function
+ docstrings)
+ level 3 : logs Out-of-date Jobs in Out-of-date Tasks, no
+ explanation
+ level 4 : logs Out-of-date Jobs in Out-of-date Tasks,
+ saying why they are out of date (include only
+ list of up-to-date tasks)
+ level 5 : All Jobs in Out-of-date Tasks (include only list
+ of up-to-date tasks)
+ level 6 : All jobs in All Tasks whether out of date or not
+
+ """
+
+ def _get_job_names(unglobbed_params, indent_str):
+ job_names = self.job_descriptor(unglobbed_params, verbose_abbreviated_path, runtime_data)[1]
+ if len(job_names) > 1:
+ job_names = ([indent_str + job_names[0]] +
+ [indent_str + " " + jn for jn in job_names[1:]])
+ else:
+ job_names = ([indent_str + job_names[0]])
+ return job_names
+
+ if not verbose:
+ return []
+
+ indent_str = ' ' * indent
+
+ messages = []
+
+ # LOGGER: level 1 : logs Out-of-date Tasks (names and warnings)
+ messages.append("Task = %r %s " % (self._get_display_name(),
+ (" >>Forced to rerun<<" if force_rerun else "")))
+ if verbose == 1:
+ return messages
+
+ # LOGGER: level 2 : logs All Tasks (including any task function
+ # docstrings)
+ if verbose >= 2 and len(self.func_description):
+ messages.append(indent_str + '"' + self.func_description + '"')
+
+ #
+ # single job state
+ #
+ if verbose >= 10:
+ if self._is_single_job_single_output == self._single_job_single_output:
+ messages.append(" Single job single output")
+ elif self._is_single_job_single_output == self._multiple_jobs_outputs:
+ messages.append(" Multiple jobs Multiple outputs")
+ else:
+ messages.append(" Single jobs status depends on %r" %
+ self._is_single_job_single_output._get_display_name())
+
+ # LOGGER: No job if less than 2
+ if verbose <= 2:
+ return messages
+
+ # increase indent for jobs up to date status
+ indent_str += " " * 3
+
+ #
+ # If has an @active_if decorator, check if the task needs to be run
+ # @active_if parameters may be call back functions or booleans
+ #
+ if not self.is_active:
+ # LOGGER
+ if verbose <= 3:
+ return messages
+ messages.append(indent_str + "Task is inactive")
+ # add spacer line
+ messages.append("")
+ return messages
+
+ #
+ # No parameters: just call task function
+ #
+ if self.param_generator_func is None:
# LOGGER
if verbose <= 3:
return messages
@@ -1177,35 +2200,40 @@ class _task (node):
# needs update func = None: always needs update
#
if not self.needs_update_func:
- messages.append(indent_str + "Task needs update: No function to check if up-to-date or not")
+ messages.append(indent_str + "Task needs update: No func to check if up-to-date.")
return messages
if self.needs_update_func == needs_update_check_modify_time:
- needs_update, msg = self.needs_update_func (task=self, job_history = job_history, verbose_abbreviated_path = verbose_abbreviated_path)
+ needs_update, msg = self.needs_update_func(
+ task=self, job_history=job_history,
+ verbose_abbreviated_path=verbose_abbreviated_path)
else:
- needs_update, msg = self.needs_update_func ()
+ needs_update, msg = self.needs_update_func()
if needs_update:
messages.append(indent_str + "Task needs update: %s" % msg)
#
# Get rid of up-to-date messages:
- # Superfluous for parts of the pipeline which are up-to-date and
- # Misleading for parts of the pipeline which require updating:
- # tasks might have to run based on dependencies anyway
+ # Superfluous for parts of the pipeline which are up-to-date
+ # Misleading for parts of the pipeline which require
+ # updating: tasks might have to run based on dependencies
+ # anyway
#
- #else:
+ # else:
# if task_is_out_of_date:
- # messages.append(indent_str + "Task appears up-to-date but will rerun after its dependencies")
+ # messages.append(indent_str + "Task appears up-to-date but
+ # will rerun after its dependencies")
# else:
# messages.append(indent_str + "Task up-to-date")
else:
runtime_data["MATCH_FAILURE"] = []
#
- # return messages description per job if verbose > 5 else whether up to date or not
+ # return messages description per job if verbose > 5 else
+ # whether up to date or not
#
cnt_jobs = 0
- for param, descriptive_param in self.param_generator_func(runtime_data):
+ for params, unglobbed_params in self.param_generator_func(runtime_data):
cnt_jobs += 1
#
@@ -1213,43 +2241,44 @@ class _task (node):
#
if not self.needs_update_func:
if verbose >= 5:
- messages.extend(get_job_names (descriptive_param, indent_str))
- messages.append(indent_str + " Jobs needs update: No function to check if up-to-date or not")
+ messages.extend(_get_job_names(unglobbed_params, indent_str))
+ messages.append(indent_str + " Jobs needs update: No "
+ "function to check if up-to-date or not")
continue
if self.needs_update_func == needs_update_check_modify_time:
- needs_update, msg = self.needs_update_func (*param, task=self, job_history = job_history, verbose_abbreviated_path = verbose_abbreviated_path)
+ needs_update, msg = self.needs_update_func(
+ *params, task=self, job_history=job_history,
+ verbose_abbreviated_path=verbose_abbreviated_path)
else:
- needs_update, msg = self.needs_update_func (*param)
+ needs_update, msg = self.needs_update_func(*params)
if needs_update:
- messages.extend(get_job_names (descriptive_param, indent_str))
+ messages.extend(_get_job_names(unglobbed_params, indent_str))
if verbose >= 4:
- per_job_messages = [(indent_str + s) for s in (" Job needs update: %s" % msg).split("\n")]
+ per_job_messages = [(indent_str + s)
+ for s in (" Job needs update: %s" % msg).split("\n")]
messages.extend(per_job_messages)
else:
messages.append(indent_str + " Job needs update")
-
# up to date: log anyway if verbose
else:
# LOGGER
if (task_is_out_of_date and verbose >= 5) or verbose >= 6:
- messages.extend(get_job_names (descriptive_param, indent_str))
+ messages.extend(_get_job_names(unglobbed_params, indent_str))
#
# Get rid of up-to-date messages:
- # Superfluous for parts of the pipeline which are up-to-date and
+ # Superfluous for parts of the pipeline which are up-to-date
# Misleading for parts of the pipeline which require updating:
# tasks might have to run based on dependencies anyway
#
- #if not task_is_out_of_date:
+ # if not task_is_out_of_date:
# messages.append(indent_str + " Job up-to-date")
-
if cnt_jobs == 0:
- messages.append(indent_str + "!!! No jobs for this task. "
- "Are you sure there is not a error in your "
- "code / regular expression?")
+ messages.append(indent_str + "!!! No jobs for this task. Are you sure there is "
+ "not a error in your code / regular expression?")
# LOGGER
if verbose >= 4 or (verbose and cnt_jobs == 0):
if runtime_data and "MATCH_FAILURE" in runtime_data:
@@ -1259,66 +2288,62 @@ class _task (node):
messages.append("")
return messages
+ # _________________________________________________________________________
-
-
- #_____________________________________________________________________________________
-
- # signal
+ # _is_up_to_date
#
+ # use to be named signal
# returns whether up to date
+ # stops recursing if true
#
- #_____________________________________________________________________________________
- def signal (self, verbose_logger_job_history):
+ # _________________________________________________________________________
+ def _is_up_to_date(self, verbose_logger_job_history):
"""
- If up to date: signal = true
If true, depth first search will not pass through this node
"""
if not verbose_logger_job_history:
raise Exception("verbose_logger_job_history is None")
- verbose_logger = verbose_logger_job_history[0]
- job_history = verbose_logger_job_history[1]
+ verbose_logger = verbose_logger_job_history[0]
+ job_history = verbose_logger_job_history[1]
try:
- logger = verbose_logger.logger
- verbose = verbose_logger.verbose
- runtime_data = verbose_logger.runtime_data
+ logger = verbose_logger.logger
+ verbose = verbose_logger.verbose
+ runtime_data = verbose_logger.runtime_data
verbose_abbreviated_path = verbose_logger.verbose_abbreviated_path
- log_at_level (logger, 10, verbose,
- " Task = " + self.get_task_name())
+ log_at_level(logger, 10, verbose, " Task = %r " % self._get_display_name())
#
# If job is inactive, always consider it up-to-date
#
- if (self.active_if_checks != None and
- any( not arg() if isinstance(arg, collections.Callable) else not arg
- for arg in self.active_if_checks)):
- log_at_level (logger, 10, verbose,
- " Inactive task: treat as Up to date")
- #print 'signaling that the inactive task is up to date'
+ if (self.active_if_checks is not None and
+ any(not arg() if isinstance(arg, collections.Callable) else not arg
+ for arg in self.active_if_checks)):
+ log_at_level(logger, 10, verbose, " Inactive task: treat as Up to date")
+ # print 'signaling that the inactive task is up to date'
return True
#
# Always needs update if no way to check if up to date
#
- if self.needs_update_func == None:
- log_at_level (logger, 10, verbose,
- " No update function: treat as out of date")
+ if self.needs_update_func is None:
+ log_at_level(logger, 10, verbose, " No update function: treat as out of date")
return False
#
# if no parameters, just return the results of needs update
#
- if self.param_generator_func == None:
+ if self.param_generator_func is None:
if self.needs_update_func:
if self.needs_update_func == needs_update_check_modify_time:
- needs_update, msg = self.needs_update_func (task=self, job_history = job_history, verbose_abbreviated_path = verbose_abbreviated_path)
+ needs_update, msg = self.needs_update_func(
+ task=self, job_history=job_history,
+ verbose_abbreviated_path=verbose_abbreviated_path)
else:
- needs_update, msg = self.needs_update_func ()
- log_at_level (logger, 10, verbose,
- " Needs update = %s" % needs_update)
+ needs_update, msg = self.needs_update_func()
+ log_at_level(logger, 10, verbose, " Needs update = %s" % needs_update)
return not needs_update
else:
return True
@@ -1326,27 +2351,29 @@ class _task (node):
#
# return not up to date if ANY jobs needs update
#
- for param, descriptive_param in self.param_generator_func(runtime_data):
+ for params, unglobbed_params in self.param_generator_func(runtime_data):
if self.needs_update_func == needs_update_check_modify_time:
- needs_update, msg = self.needs_update_func (*param, task=self, job_history = job_history, verbose_abbreviated_path = verbose_abbreviated_path)
+ needs_update, msg = self.needs_update_func(
+ *params, task=self, job_history=job_history,
+ verbose_abbreviated_path=verbose_abbreviated_path)
else:
- needs_update, msg = self.needs_update_func (*param)
+ needs_update, msg = self.needs_update_func(*params)
if needs_update:
- log_at_level (logger, 10, verbose, " Needing update:\n %s" % self.get_job_name(descriptive_param, verbose_abbreviated_path, runtime_data))
+ log_at_level(logger, 10, verbose, " Needing update:\n %s"
+ % self._get_job_name(unglobbed_params,
+ verbose_abbreviated_path, runtime_data))
return False
#
# Percolate warnings from parameter factories
#
if (verbose >= 1 and "ruffus_WARNING" in runtime_data and
- self.param_generator_func in runtime_data["ruffus_WARNING"]):
+ self.param_generator_func in runtime_data["ruffus_WARNING"]):
for msg in runtime_data["ruffus_WARNING"][self.param_generator_func]:
- logger.warning(" 'In Task %s' %s " % (self.get_task_name(True), msg))
-
- log_at_level (logger, 10, verbose, " All jobs up to date")
-
-
+ logger.warning(" 'In Task\n%s\n%s" % (
+ self.description_with_args_placeholder % "...", msg))
+ log_at_level(logger, 10, verbose, " All jobs up to date")
return True
@@ -1354,13 +2381,12 @@ class _task (node):
# removed for compatibility with python 3.x
#
# rethrow exception after adding task name
- #except error_task, inst:
+ # except error_task, inst:
# inst.specify_task(self, "Exceptions in dependency checking")
# raise
except:
exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
-
#
# rethrow exception after adding task name
#
@@ -1369,35 +2395,33 @@ class _task (node):
inst.specify_task(self, "Exceptions in dependency checking")
raise
- exception_stack = traceback.format_exc()
- exception_name = exceptionType.__module__ + '.' + exceptionType.__name__
- exception_value = str(exceptionValue)
+ exception_stack = traceback.format_exc()
+ exception_name = exceptionType.__module__ + '.' + exceptionType.__name__
+ exception_value = str(exceptionValue)
if len(exception_value):
exception_value = "(%s)" % exception_value
errt = RethrownJobError([(self._name,
- "",
- exception_name,
- exception_value,
- exception_stack)])
+ "",
+ exception_name,
+ exception_value,
+ exception_stack)])
errt.specify_task(self, "Exceptions generating parameters")
raise errt
+ # _________________________________________________________________________
-
- #_____________________________________________________________________________________
-
- # get_output_files
+ # _get_output_files
#
#
- #_____________________________________________________________________________________
- def get_output_files (self, do_not_expand_single_job_tasks, runtime_data):
+ # _________________________________________________________________________
+ def _get_output_files(self, do_not_expand_single_job_tasks, runtime_data):
"""
Cache output files
- If flattened is True, returns file as a list of strings,
- flattening any nested structures and discarding non string names
- Normally returns a list with one item for each job or a just a list of file names.
- For "single_job_single_output" i.e. @merge and @files with single jobs,
+ Normally returns a list with one item for each job or a just a list
+ of file names.
+ For "_single_job_single_output"
+ i.e. @merge and @files with single jobs,
returns the output of a single job (i.e. can be a string)
"""
@@ -1405,44 +2429,41 @@ class _task (node):
# N.B. active_if_checks is called once per task
# in make_job_parameter_generator() for consistency
#
- # self.is_active can be set using self.active_if_checks in that function,
- # and therefore can be changed BETWEEN invocations of pipeline_run
+ # self.is_active can be set using self.active_if_checks in that
+ # function, and therefore can be changed BETWEEN invocations
+ # of pipeline_run
#
# self.is_active is not used anywhere else
#
if (not self.is_active):
return []
- #
- # This looks like the wrong place to flatten
- #
- flattened = False
- if self.output_filenames == None:
+ if self.output_filenames is None:
self.output_filenames = []
# skip tasks which don't have parameters
- if self.param_generator_func != None:
+ if self.param_generator_func is not None:
cnt_jobs = 0
- for param, descriptive_param in self.param_generator_func(runtime_data):
+ for params, unglobbed_params in self.param_generator_func(runtime_data):
cnt_jobs += 1
# skip tasks which don't have output parameters
- if len(param) >= 2:
- # make sure each @split or @subdivide or @originate returns a list of jobs
- # i.e. each @split or @subdivide or @originate is always a ->many operation
+ if len(params) >= 2:
+ # make sure each @split or @subdivide or @originate
+ # returns a list of jobs
+ # i.e. each @split or @subdivide or @originate is
+ # always a ->many operation
# even if len(many) can be 1 (or zero)
- if self.indeterminate_output and not non_str_sequence(param[1]):
- self.output_filenames.append([param[1]])
+ if self.indeterminate_output and not non_str_sequence(params[1]):
+ self.output_filenames.append([params[1]])
else:
- self.output_filenames.append(param[1])
-
+ self.output_filenames.append(params[1])
- if self._single_job_single_output == self.single_job_single_output:
+ if self._is_single_job_single_output == self._single_job_single_output:
if cnt_jobs > 1:
- raise error_task_get_output(self,
- "Task which is supposed to produce a single output "
- "somehow has more than one job.")
+ raise error_task_get_output(self, "Task which is supposed to produce a "
+ "single output somehow has more than one job.")
#
# The output of @split should be treated as multiple jobs
@@ -1457,21 +2478,23 @@ class _task (node):
# So len(list) = 1
#
# 2) The output of each @split job is a list
- # The items in this list of lists are each a job in subsequent tasks
+ # The items in this list of lists are each a job in
+ # subsequent tasks
#
#
- # So we need to concatenate these separate lists into a single list of output
+ # So we need to concatenate these separate lists into a
+ # single list of output
#
# For example:
# @split(["a.1", "b.1"], regex(r"(.)\.1"), r"\1.*.2")
# def example(input, output):
- # # JOB 1
- # # a.1 -> a.i.2
- # # -> a.j.2
+ # JOB 1
+ # a.1 -> a.i.2
+ # -> a.j.2
#
- # # JOB 2
- # # b.1 -> b.i.2
- # # -> b.j.2
+ # JOB 2
+ # b.1 -> b.i.2
+ # -> b.j.2
#
# output_filenames = [ [a.i.2, a.j.2], [b.i.2, b.j.2] ]
#
@@ -1481,49 +2504,35 @@ class _task (node):
#
# @split("a.1", r"a.*.2")
# def example(input, output):
- # # only job
- # # a.1 -> a.i.2
- # # -> a.j.2
+ # only job
+ # a.1 -> a.i.2
+ # -> a.j.2
#
# output_filenames = [ [a.i.2, a.j.2] ]
#
# we want [ a.i.2, a.j.2 ]
#
if len(self.output_filenames) and self.indeterminate_output:
- self.output_filenames = reduce(lambda x,y: x + y, self.output_filenames)
-
-
- if flattened:
- # if single file name, return that
- # accepts unicode
- if (do_not_expand_single_job_tasks and
- len(self.output_filenames) and
- isinstance(self.output_filenames[0], path_str_type)):
- return self.output_filenames
- # if it is flattened, might as well sort it
- return sorted(get_strings_in_nested_sequence(self.output_filenames))
-
- else:
- # special handling for jobs which have a single task,
- if (do_not_expand_single_job_tasks and
- self._single_job_single_output and
- len(self.output_filenames) ):
- return self.output_filenames[0]
-
- #
- # sort by jobs so it is just a weeny little bit less deterministic
- #
- return sorted(self.output_filenames, key = lambda x: str(x))
+ self.output_filenames = reduce(lambda x, y: x + y, self.output_filenames)
+ # special handling for jobs which have a single task
+ if (do_not_expand_single_job_tasks and
+ self._is_single_job_single_output and
+ len(self.output_filenames)):
+ return self.output_filenames[0]
+ #
+ # sort by jobs so it is just a weeny little bit less deterministic
+ #
+ return sorted(self.output_filenames, key=lambda x: str(x))
- #_____________________________________________________________________________________
+ # _________________________________________________________________________
- # completed
+ # _completed
#
# All logging logic moved to caller site
- #_____________________________________________________________________________________
- def completed (self):
+ # _________________________________________________________________________
+ def _completed(self):
"""
called even when all jobs are up to date
"""
@@ -1535,32 +2544,31 @@ class _task (node):
f()
#
- # indeterminate output. Check actual output again if someother tasks job function depend on it
+ # indeterminate output. Check actual output again if someother tasks
+ # job function depend on it
# used for @split
#
if self.indeterminate_output:
self.output_filenames = None
+ # _________________________________________________________________________
+ # _handle_tasks_globs_in_inputs
-
-
-
-
-
-
- #_________________________________________________________________________________________
-
- # handle_tasks_globs_in_inputs
-
- #_________________________________________________________________________________________
- def handle_tasks_globs_in_inputs(self, input_params):
+ # _________________________________________________________________________
+ def _handle_tasks_globs_in_inputs(self, input_params, modify_inputs_mode):
"""
Helper function for tasks which
1) Notes globs and tasks
2) Replaces tasks names and functions with actual tasks
3) Adds task dependencies automatically via task_follows
+
+ modify_inputs_mode = results["modify_inputs_mode"] =
+ t_extra_inputs.ADD_TO_INPUTS | REPLACE_INPUTS |
+ KEEP_INPUTS | KEEP_OUTPUTS
"""
+ # DEBUGGG
+ #print(" task._handle_tasks_globs_in_inputs start %s" % (self._get_display_name(), ), file = sys.stderr)
#
# get list of function/function names and globs
#
@@ -1569,17 +2577,79 @@ class _task (node):
#
# replace function / function names with tasks
#
- tasks = self.task_follows(function_or_func_names)
- functions_to_tasks = dict(zip(function_or_func_names, tasks))
- input_params = replace_func_names_with_tasks(input_params, functions_to_tasks)
+ if modify_inputs_mode == t_extra_inputs.ADD_TO_INPUTS:
+ description_with_args_placeholder = \
+ self.description_with_args_placeholder % "add_inputs = add_inputs(%r)"
+ elif modify_inputs_mode == t_extra_inputs.REPLACE_INPUTS:
+ description_with_args_placeholder = \
+ self.description_with_args_placeholder % "replace_inputs = add_inputs(%r)"
+ elif modify_inputs_mode == t_extra_inputs.KEEP_OUTPUTS:
+ description_with_args_placeholder = \
+ self.description_with_args_placeholder % "output =%r"
+ else: # t_extra_inputs.KEEP_INPUTS
+ description_with_args_placeholder = \
+ self.description_with_args_placeholder % "input =%r"
+
+ tasks = self._connect_parents(description_with_args_placeholder, True, function_or_func_names)
+ functions_to_tasks = dict()
+ for funct_name_task_or_pipeline, task in zip(function_or_func_names, tasks):
+ if isinstance(funct_name_task_or_pipeline, Pipeline):
+ functions_to_tasks["PIPELINE=%s=PIPELINE" % funct_name_task_or_pipeline.name] = task
+ else:
+ functions_to_tasks[funct_name_task_or_pipeline] = task
+ # replace strings, tasks, pipelines with tasks
+ input_params = replace_placeholders_with_tasks_in_input_params(input_params, functions_to_tasks)
+ #DEBUGGG
+ #print(" task._handle_tasks_globs_in_inputs finish %s" % (self._get_display_name(), ), file = sys.stderr)
return t_params_tasks_globs_run_time_data(input_params, tasks, globs, runtime_data_names)
+ # _________________________________________________________________________
+ # _choose_file_names_transform
+ # _________________________________________________________________________
+ def _choose_file_names_transform(self, parsed_args,
+ valid_tags=(regex, suffix, formatter)):
+ """
+ shared code for subdivide, transform, product etc for choosing method
+ for transform input file to output files
+ """
+ file_name_transform_tag = parsed_args["filter"]
+ valid_tag_names = []
+ # regular expression match
+ if (regex in valid_tags):
+ valid_tag_names.append("regex()")
+ if isinstance(file_name_transform_tag, regex):
+ return t_regex_file_names_transform(self,
+ file_name_transform_tag,
+ self.error_type,
+ self.syntax)
+
+ # simulate end of string (suffix) match
+ if (suffix in valid_tags):
+ valid_tag_names.append("suffix()")
+ if isinstance(file_name_transform_tag, suffix):
+ output_dir = parsed_args["output_dir"] if "output_dir" in parsed_args else []
+ return t_suffix_file_names_transform(self,
+ file_name_transform_tag,
+ self.error_type,
+ self.syntax,
+ output_dir)
+ # new style string.format()
+ if (formatter in valid_tags):
+ valid_tag_names.append("formatter()")
+ if isinstance(file_name_transform_tag, formatter):
+ return t_formatter_file_names_transform(self,
+ file_name_transform_tag,
+ self.error_type,
+ self.syntax)
+ raise self.error_type(self,
+ "%s expects one of %s as the second argument"
+ % (self.syntax, ", ".join(valid_tag_names)))
- #8888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+ # 8888888888888888888888888888888888888888888888888888888888888888888888888
# task handlers
@@ -1589,1190 +2659,1566 @@ class _task (node):
# 3) needs_update_func
# 4) job wrapper
+ # 8888888888888888888888888888888888888888888888888888888888888888888888888
- #8888888888888888888888888888888888888888888888888888888888888888888888888888888888888
- #_________________________________________________________________________________________
-
- # do_task_subdivide
-
- #_________________________________________________________________________________________
- def do_task_subdivide (self, orig_args, decorator_name, error_type):
+ def _do_nothing_setup(self):
"""
- @subdivide and @split are synonyms
- Common code here
+ Task is already set up: do nothing
"""
+ return set()
- if len(orig_args) < 3:
- raise error_type(self, "Too few arguments for %s" % decorator_name)
+ # ========================================================================
+ # _decorator_originate
+ # originate does have an Input param.
+ # It is just None (and not set-able)
+ # ========================================================================
+ def _decorator_originate(self, *unnamed_args, **named_args):
+ """
+ @originate
+ """
+ self.syntax = "@originate"
+ self.description_with_args_placeholder = "%s(%%s)\n%s" % (self.syntax,
+ self._get_decorated_function())
+ self._prepare_originate(unnamed_args, named_args)
- #
- # replace function / function names with tasks
- #
- input_files_task_globs = self.handle_tasks_globs_in_inputs(orig_args[0])
+ # originate
+ # self.has_input_param = True
- # allows split to take a single file or task
- input_files_task_globs.single_file_to_list()
-
- # how to transform input to output file name
- file_names_transform = self.choose_file_names_transform (orig_args[1], error_type, decorator_name)
+ # _________________________________________________________________________
- orig_args = orig_args[2:]
+ # _prepare_originate
- # inputs can also be defined by pattern match
- extra_inputs, replace_inputs, output_pattern, extra_params = self.get_extra_inputs_outputs_extra (orig_args, error_type, decorator_name)
+ # _________________________________________________________________________
+ def _prepare_originate(self, unnamed_args, named_args):
+ """
+ Common function for pipeline.originate and @originate
+ """
+ self.error_type = error_task_originate
+ self._set_action_type(Task._action_task_originate)
+ self._setup_task_func = Task._originate_setup
+ self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
+ self.job_wrapper = job_wrapper_output_files
+ self.job_descriptor = io_files_one_to_many_job_descriptor
+ self.single_multi_io = self._many_to_many
+ # output is not a glob
+ self.indeterminate_output = 0
#
- # output globs will be replaced with files. But there should not be tasks here!
+ # Parse named and unnamed arguments
#
- output_files_task_globs = self.handle_tasks_globs_in_inputs(output_pattern)
- if len(output_files_task_globs.tasks):
- raise error_type(self, ("%s cannot output to another task. "
- "Do not include tasks in output parameters.") % decorator_name)
-
-
-
- self.param_generator_func = subdivide_param_factory ( input_files_task_globs,
- False, # flatten input
- file_names_transform,
- extra_inputs,
- replace_inputs,
- output_files_task_globs,
- *extra_params)
- self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
- self.job_wrapper = job_wrapper_io_files
- #self.job_descriptor = io_files_job_descriptor # (orig_args[0], output_runtime_data_names)
- self.job_descriptor = io_files_one_to_many_job_descriptor
-
- # output is a glob
- self.indeterminate_output = 2
- self.single_multi_io = self.many_to_many
-
- #_________________________________________________________________________________________
+ self.parsed_args = parse_task_arguments(unnamed_args, named_args, ["output", "extras"],
+ self.description_with_args_placeholder)
- # task_split
+ # _________________________________________________________________________
- #_________________________________________________________________________________________
- def do_task_simple_split (self, orig_args, decorator_name, error_type):
-
- #check enough arguments
- if len(orig_args) < 2:
- raise error_type(self, "Too few arguments for %s" % decorator_name)
+ # _originate_setup
+ # _________________________________________________________________________
+ def _originate_setup(self):
+ """
+ Finish setting up originate
+ """
#
- # replace function / function names with tasks
+ # If self.parsed_args["output"] is a single item (e.g. file name),
+ # that will be treated as a list
+ # Each item in the list of these will be called as an output in a
+ # separate function call
#
- input_files_task_globs = self.handle_tasks_globs_in_inputs(orig_args[0])
+ output_params = self.parsed_args["output"]
+ if not non_str_sequence(output_params):
+ output_params = [output_params]
#
- # replace output globs with files
+ # output globs will be replaced with files. But there should not be
+ # tasks here!
#
- output_files_task_globs = self.handle_tasks_globs_in_inputs(orig_args[1])
- if len(output_files_task_globs.tasks):
- raise error_type(self, ("%s cannot output to another task. "
- "Do not include tasks in output parameters.") % decorator_name)
-
- extra_params = orig_args[2:]
- self.param_generator_func = split_param_factory (input_files_task_globs, output_files_task_globs, *extra_params)
+ list_output_files_task_globs = [self._handle_tasks_globs_in_inputs(
+ oo, t_extra_inputs.KEEP_INPUTS) for oo in output_params]
+ for oftg in list_output_files_task_globs:
+ if len(oftg.tasks):
+ raise self.error_type(self, "%s cannot output to another "
+ "task. Do not include tasks in "
+ "output parameters." % self.syntax)
+ self.param_generator_func = originate_param_factory(list_output_files_task_globs,
+ *self.parsed_args["extras"])
+ return set()
- self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
- self.job_wrapper = job_wrapper_io_files
- #self.job_descriptor = io_files_job_descriptor# (orig_args[1], output_runtime_data_names)
- self.job_descriptor = io_files_one_to_many_job_descriptor
-
- # output is a glob
- self.indeterminate_output = 1
- self.single_multi_io = self.one_to_many
+ # ========================================================================
+ # _decorator_transform
+ # ========================================================================
+ def _decorator_transform(self, *unnamed_args, **named_args):
+ """
+ @originate
+ """
+ self.syntax = "@transform"
+ self.description_with_args_placeholder = "%s(%%s)\n%s" % (
+ self.syntax, self._get_decorated_function())
+ self._prepare_transform(unnamed_args, named_args)
- #_________________________________________________________________________________________
+ # _________________________________________________________________________
- # task_split
+ # _prepare_transform
- #_________________________________________________________________________________________
- def task_split (self, orig_args):
+ # _________________________________________________________________________
+ def _prepare_transform(self, unnamed_args, named_args):
"""
- Splits a single set of input files into multiple output file names,
- where the number of output files may not be known beforehand.
+ Common function for pipeline.transform and @transform
"""
- decorator_name = "@split"
- error_type = error_task_split
- self.set_action_type (_task.action_task_split)
+ self.error_type = error_task_transform
+ self._set_action_type(Task._action_task_transform)
+ self._setup_task_func = Task._transform_setup
+ self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
+ self.job_wrapper = job_wrapper_io_files
+ self.job_descriptor = io_files_job_descriptor
+ self.single_multi_io = self._many_to_many
#
- # This is actually @subdivide
- #
- if isinstance(orig_args[1], regex):
- self.do_task_subdivide(orig_args, decorator_name, error_type)
-
- #
- # This is actually @split
+ # Parse named and unnamed arguments
#
- else:
- self.do_task_simple_split(orig_args, decorator_name, error_type)
-
+ self.parsed_args = parse_task_arguments(unnamed_args, named_args,
+ ["input", "filter", "modify_inputs",
+ "output", "extras", "output_dir"],
+ self.description_with_args_placeholder)
+ # _________________________________________________________________________
- #_________________________________________________________________________________________
+ # _transform_setup
- # task_originate
-
- #_________________________________________________________________________________________
- def task_originate (self, orig_args):
+ # _________________________________________________________________________
+ def _transform_setup(self):
"""
- Splits out multiple output file names,
- where the number of output files may or may not be known beforehand.
- This is a synonym for @split(None,...)
+ Finish setting up transform
"""
- decorator_name = "@originate"
- error_type = error_task_originate
- self.set_action_type (_task.action_task_originate)
-
- if len(orig_args) < 1:
- raise error_type(self, "%s takes a single argument" % decorator_name)
+ #DEBUGGG
+ #print(" task._transform_setup start %s" % (self._get_display_name(), ), file = sys.stderr)
- output_params = orig_args[0]
+ #
+ # replace function / function names with tasks
+ #
+ input_files_task_globs = self._handle_tasks_globs_in_inputs(self.parsed_args["input"],
+ t_extra_inputs.KEEP_INPUTS)
+ ancestral_tasks = set(input_files_task_globs.tasks)
- # make sure output_params is a list.
- # Each of these will be called as an output
- if not non_str_sequence (output_params):
- output_params = [output_params]
+ # _____________________________________________________________________
+ #
+ # _single_job_single_output is bad policy. Can we remove it?
+ # What does this actually mean in Ruffus semantics?
+ #
+ #
+ # allows transform to take a single file or task
+ if input_files_task_globs.single_file_to_list():
+ self._is_single_job_single_output = self._single_job_single_output
#
- # output globs will be replaced with files. But there should not be tasks here!
+ # whether transform generates a list of jobs or not will depend on
+ # the parent task
#
- list_output_files_task_globs = [self.handle_tasks_globs_in_inputs(oo) for oo in output_params]
- for oftg in list_output_files_task_globs:
- if len(oftg.tasks):
- raise error_type(self, ("%s cannot output to another task. "
- "Do not include tasks in output parameters.") % decorator_name)
+ elif isinstance(input_files_task_globs.params, Task):
+ self._is_single_job_single_output = input_files_task_globs.params
- self.param_generator_func = originate_param_factory (list_output_files_task_globs, orig_args[1:])
- self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
- self.job_wrapper = job_wrapper_output_files
- self.job_descriptor = io_files_one_to_many_job_descriptor
+ # _____________________________________________________________________
- # output is not a glob
- self.indeterminate_output = 0
- self.single_multi_io = self.many_to_many
+ # how to transform input to output file name
+ file_names_transform = self._choose_file_names_transform(self.parsed_args)
+ modify_inputs = self.parsed_args["modify_inputs"]
+ if modify_inputs is not None:
+ modify_inputs = self._handle_tasks_globs_in_inputs(
+ modify_inputs, self.parsed_args["modify_inputs_mode"])
+ ancestral_tasks = ancestral_tasks.union(modify_inputs.tasks)
+ self.param_generator_func = transform_param_factory(input_files_task_globs,
+ file_names_transform,
+ modify_inputs,
+ self.parsed_args["modify_inputs_mode"],
+ self.parsed_args["output"],
+ *self.parsed_args["extras"])
+ #DEBUGGG
+ #print(" task._transform_setup finish %s" % (self._get_display_name(), ), file = sys.stderr)
+ return ancestral_tasks
+ # ========================================================================
+ # _decorator_subdivide
+ # ========================================================================
+ def _decorator_subdivide(self, *unnamed_args, **named_args):
+ """
+ @subdivide
+ """
+ self.syntax = "@subdivide"
+ self.description_with_args_placeholder = "%s(%%s)\n%s" % (self.syntax,
+ self._get_decorated_function())
+ self._prepare_subdivide(unnamed_args, named_args)
- #_________________________________________________________________________________________
+ # _________________________________________________________________________
- # task_subdivide
+ # _prepare_subdivide
- #_________________________________________________________________________________________
- def task_subdivide (self, orig_args):
+ # _________________________________________________________________________
+ def _prepare_subdivide(self, unnamed_args, named_args):
"""
- Splits a single set of input files into multiple output file names,
- where the number of output files may not be known beforehand.
+ Common code for @subdivide and pipeline.subdivide
+ @split can also end up here
"""
- decorator_name = "@subdivide"
- error_type = error_task_subdivide
- self.set_action_type (_task.action_task_subdivide)
- self.do_task_subdivide(orig_args, decorator_name, error_type)
+ self.error_type = error_task_subdivide
+ self._set_action_type(Task._action_task_subdivide)
+ self._setup_task_func = Task._subdivide_setup
+ self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
+ self.job_wrapper = job_wrapper_io_files
+ self.job_descriptor = io_files_one_to_many_job_descriptor
+ self.single_multi_io = self._many_to_many
+ # output is a glob
+ self.indeterminate_output = 2
- #_________________________________________________________________________________________
+ #
+ # Parse named and unnamed arguments
+ #
+ self.parsed_args = parse_task_arguments(unnamed_args, named_args,
+ ["input", "filter", "modify_inputs",
+ "output", "extras"],
+ self.description_with_args_placeholder)
+
+ # _________________________________________________________________________
- # get_extra_inputs
+ # _subdivide_setup
- #_________________________________________________________________________________________
- def get_extra_inputs_outputs_extra (self, orig_args, error_type, decorator_name):
+ # _________________________________________________________________________
+ def _subdivide_setup(self):
"""
- shared code for subdivide, transform, product etc for parsing orig_args into
- add_inputs/inputs, output, extra
+ Finish setting up subdivide
"""
#
- # inputs can also be defined by pattern match
+ # replace function / function names with tasks
#
- if isinstance(orig_args[0], inputs):
- if len(orig_args) < 2:
- raise error_type(self, "Too few arguments for %s" % decorator_name)
- if len(orig_args[0].args) != 1:
- raise error_task_transform_inputs_multiple_args(self,
- "inputs(...) expects only a single argument. "
- "This can be, for example, a file name, "
- "a regular expression pattern, or any "
- "nested structure. If the intention was to "
- "specify a tuple as the input parameter, "
- "please wrap the elements of the tuple "
- "in brackets in the decorator\n\n"
- "%s(..., inputs(...), ...)\n" % (decorator_name))
- replace_inputs = t_extra_inputs.REPLACE_INPUTS
- extra_inputs = self.handle_tasks_globs_in_inputs(orig_args[0].args[0])
- output_pattern = orig_args[1]
- extra_params = orig_args[2:]
- elif isinstance(orig_args[0], add_inputs):
- if len(orig_args) < 2:
- raise error_type(self, "Too few arguments for %s" % decorator_name)
- replace_inputs = t_extra_inputs.ADD_TO_INPUTS
- extra_inputs = self.handle_tasks_globs_in_inputs(orig_args[0].args)
- output_pattern = orig_args[1]
- extra_params = orig_args[2:]
- else:
- replace_inputs = t_extra_inputs.KEEP_INPUTS
- extra_inputs = None
- output_pattern = orig_args[0]
- extra_params = orig_args[1:]
+ input_files_task_globs = self._handle_tasks_globs_in_inputs(self.parsed_args["input"],
+ t_extra_inputs.KEEP_INPUTS)
- return extra_inputs, replace_inputs, output_pattern, extra_params
+ # allows split to take a single file or task
+ input_files_task_globs.single_file_to_list()
- #_________________________________________________________________________________________
+ ancestral_tasks = set(input_files_task_globs.tasks)
- # choose_file_names_transform
+ # how to transform input to output file name
+ file_names_transform = self._choose_file_names_transform(self.parsed_args)
- #_________________________________________________________________________________________
- def choose_file_names_transform (self, file_name_transform_tag, error_type, decorator_name, valid_tags = (regex, suffix, formatter)):
- """
- shared code for subdivide, transform, product etc for choosing method for transform input file to output files
- """
- valid_tag_names = [];
- # regular expression match
- if (regex in valid_tags):
- valid_tag_names.append("regex()")
- if isinstance(file_name_transform_tag, regex):
- return t_regex_file_names_transform(self, file_name_transform_tag, error_type, decorator_name)
+ modify_inputs = self.parsed_args["modify_inputs"]
+ if modify_inputs is not None:
+ modify_inputs = self._handle_tasks_globs_in_inputs(
+ modify_inputs, self.parsed_args["modify_inputs_mode"])
+ ancestral_tasks = ancestral_tasks.union(modify_inputs.tasks)
- # simulate end of string (suffix) match
- if (suffix in valid_tags):
- valid_tag_names.append("suffix()")
- if isinstance(file_name_transform_tag, suffix):
- return t_suffix_file_names_transform(self, file_name_transform_tag, error_type, decorator_name)
+ #
+ # output globs will be replaced with files.
+ # But there should not be tasks here!
+ #
+ output_files_task_globs = self._handle_tasks_globs_in_inputs(self.parsed_args["output"],
+ t_extra_inputs.KEEP_OUTPUTS)
+ if len(output_files_task_globs.tasks):
+ raise self.error_type(self, ("%s cannot output to another task. Do not include tasks "
+ "in output parameters.") % self.syntax)
- # new style string.format()
- if (formatter in valid_tags):
- valid_tag_names.append("formatter()")
- if isinstance(file_name_transform_tag, formatter):
- return t_formatter_file_names_transform(self, file_name_transform_tag, error_type, decorator_name)
+ self.param_generator_func = subdivide_param_factory(input_files_task_globs,
+ # False, #
+ # flatten input
+ # removed
+ file_names_transform,
+ modify_inputs,
+ self.parsed_args["modify_inputs_mode"],
+ output_files_task_globs,
+ *self.parsed_args["extras"])
+ return ancestral_tasks
- raise error_type(self, "%s expects one of %s as the second argument" % (decorator_name, ", ".join(valid_tag_names)))
+ # ========================================================================
+
+ # _decorator_split
+
+ # ========================================================================
+ def _decorator_split(self, *unnamed_args, **named_args):
+ """
+ @split
+ """
+ self.syntax = "@split"
+ self.description_with_args_placeholder = "%s(%%s)\n%s" % (self.syntax,
+ self._get_decorated_function())
+
+ #
+ # This is actually @subdivide
+ #
+ if isinstance(unnamed_args[1], regex):
+ self._prepare_subdivide(unnamed_args, named_args,
+ self.description_with_args_placeholder)
+ #
+ # This is actually @split
+ #
+ else:
+ self._prepare_split(unnamed_args, named_args)
- #_________________________________________________________________________________________
+ # _________________________________________________________________________
- # task_product
+ # _prepare_split
- #_________________________________________________________________________________________
- def task_product(self, orig_args):
+ # _________________________________________________________________________
+ def _prepare_split(self, unnamed_args, named_args):
"""
- all versus all
+ Common code for @split and pipeline.split
"""
- decorator_name = "@product"
- error_type = error_task_product
- if len(orig_args) < 3:
- raise error_type(self, "Too few arguments for %s" % decorator_name)
+ self.error_type = error_task_split
+ self._set_action_type(Task._action_task_split)
+ self._setup_task_func = Task._split_setup
+ self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
+ self.job_wrapper = job_wrapper_io_files
+ self.job_descriptor = io_files_one_to_many_job_descriptor
+ self.single_multi_io = self._one_to_many
+ # output is a glob
+ self.indeterminate_output = 1
#
- # get all pairs of tasks / globs and formatter()
+ # Parse named and unnamed arguments
#
- list_input_files_task_globs = []
- list_formatter = []
- while len(orig_args) >= 3:
- if isinstance(orig_args[1], formatter):
- list_input_files_task_globs .append(orig_args[0])
- list_formatter .append(orig_args[1])
- orig_args = orig_args[2:]
- else:
- break
+ self.parsed_args = parse_task_arguments(unnamed_args, named_args,
+ ["input", "output", "extras"],
+ self.description_with_args_placeholder)
- if not len(list_formatter):
- raise error_task_product(self, "@product expects formatter() as the second argument")
+ # _________________________________________________________________________
+ # _split_setup
- self.set_action_type (_task.action_task_product)
+ # _________________________________________________________________________
+ def _split_setup(self):
+ """
+ Finish setting up split
+ """
#
# replace function / function names with tasks
#
- list_input_files_task_globs = [self.handle_tasks_globs_in_inputs(ii) for ii in list_input_files_task_globs]
-
-
- # list of new style string.format()
- file_names_transform = t_nested_formatter_file_names_transform(self, list_formatter, error_task_product, decorator_name)
-
+ input_files_task_globs = self._handle_tasks_globs_in_inputs(self.parsed_args["input"],
+ t_extra_inputs.KEEP_INPUTS)
#
- # inputs can also be defined by pattern match
+ # output globs will be replaced with files.
+ # But there should not be tasks here!
#
- extra_inputs, replace_inputs, output_pattern, extra_params = self.get_extra_inputs_outputs_extra (orig_args, error_type, decorator_name)
+ output_files_task_globs = self._handle_tasks_globs_in_inputs(self.parsed_args["output"],
+ t_extra_inputs.KEEP_OUTPUTS)
+ if len(output_files_task_globs.tasks):
+ raise self.error_type(self, "%s cannot output to another task. "
+ "Do not include tasks in output "
+ "parameters." % self.syntax)
- self.param_generator_func = product_param_factory ( list_input_files_task_globs,
- False, # flatten input
- file_names_transform,
- extra_inputs,
- replace_inputs,
- output_pattern,
- *extra_params)
- self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
- self.job_wrapper = job_wrapper_io_files
- self.job_descriptor = io_files_job_descriptor
- self.single_multi_io = self.many_to_many
+ self.param_generator_func = split_param_factory(input_files_task_globs,
+ output_files_task_globs,
+ *self.parsed_args["extras"])
+ return set(input_files_task_globs.tasks)
+ # ========================================================================
- #_________________________________________________________________________________________
+ # _decorator_merge
- # task_combinatorics
+ # ========================================================================
+ def _decorator_merge(self, *unnamed_args, **named_args):
+ """
+ @merge
+ """
+ self.syntax = "@merge"
+ self.description_with_args_placeholder = "%s(%%s)\n%s" % (self.syntax,
+ self._get_decorated_function())
+ self._prepare_merge(unnamed_args, named_args)
+
+ # _________________________________________________________________________
+
+ # _prepare_merge
- #_________________________________________________________________________________________
- def task_combinatorics (self, orig_args, combinatorics_type, decorator_name, error_type):
+ # _________________________________________________________________________
+ def _prepare_merge(self, unnamed_args, named_args):
"""
- Common code for task_permutations, task_combinations_with_replacement, task_combinations
+ Common code for @merge and pipeline.merge
"""
+ self.error_type = error_task_merge
+ self._set_action_type(Task._action_task_merge)
+ self._setup_task_func = Task._merge_setup
+ self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
+ self.job_wrapper = job_wrapper_io_files
+ self.job_descriptor = io_files_job_descriptor
+ self.single_multi_io = self._many_to_one
+ self._is_single_job_single_output = self._single_job_single_output
- if len(orig_args) < 4:
- raise error_type(self, "Too few arguments for %s" % decorator_name)
+ #
+ # Parse named and unnamed arguments
+ #
+ self.parsed_args = parse_task_arguments(unnamed_args, named_args,
+ ["input", "output", "extras"],
+ self.description_with_args_placeholder)
+ # _________________________________________________________________________
- if not isinstance(orig_args[1], formatter):
- raise error_task_product(self, "%s expects formatter() as the second argument" % decorator_name)
+ # _merge_setup
+ # _________________________________________________________________________
+ def _merge_setup(self):
+ """
+ Finish setting up merge
+ """
#
# replace function / function names with tasks
#
- input_files_task_globs = self.handle_tasks_globs_in_inputs(orig_args[0])
+ input_files_task_globs = self._handle_tasks_globs_in_inputs(self.parsed_args["input"],
+ t_extra_inputs.KEEP_INPUTS)
- k_tuple = orig_args[2]
+ self.param_generator_func = merge_param_factory(input_files_task_globs,
+ self.parsed_args["output"],
+ *self.parsed_args["extras"])
+ return set(input_files_task_globs.tasks)
- # how to transform input to output file name: len(k-tuples) of (identical) formatters
- file_names_transform = t_nested_formatter_file_names_transform(self, [orig_args[1]] * k_tuple, error_type, decorator_name)
+ # ========================================================================
+ # _decorator_collate
- self.set_action_type (_task.action_task_permutations)
-
- if not isinstance(orig_args[2], int):
- raise error_task_product(self, "%s expects an integer number as the third argument specifying the number of elements in each tuple." % decorator_name)
+ # ========================================================================
+ def _decorator_collate(self, *unnamed_args, **named_args):
+ """
+ @collate
+ """
+ self.syntax = "@collate"
+ self.description_with_args_placeholder = "%s(%%s)\n%s" % (self.syntax,
+ self._get_decorated_function())
+ self._prepare_collate(unnamed_args, named_args)
+ # _________________________________________________________________________
- orig_args = orig_args[3:]
+ # _prepare_collate
+ # _________________________________________________________________________
+ def _prepare_collate(self, unnamed_args, named_args):
+ """
+ Common code for @collate and pipeline.collate
+ """
+ self.error_type = error_task_collate
+ self._set_action_type(Task._action_task_collate)
+ self._setup_task_func = Task._collate_setup
+ self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
+ self.job_wrapper = job_wrapper_io_files
+ self.job_descriptor = io_files_job_descriptor
+ self.single_multi_io = self._many_to_many
#
- # inputs can also be defined by pattern match
+ # Parse named and unnamed arguments
#
- extra_inputs, replace_inputs, output_pattern, extra_params = self.get_extra_inputs_outputs_extra (orig_args, error_type, decorator_name)
-
- self.param_generator_func = combinatorics_param_factory ( input_files_task_globs,
- False, # flatten input
- combinatorics_type,
- k_tuple,
- file_names_transform,
- extra_inputs,
- replace_inputs,
- output_pattern,
- *extra_params)
- self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
- self.job_wrapper = job_wrapper_io_files
- self.job_descriptor = io_files_job_descriptor
- self.single_multi_io = self.many_to_many
+ self.parsed_args = parse_task_arguments(unnamed_args, named_args,
+ ["input", "filter", "modify_inputs",
+ "output", "extras"],
+ self.description_with_args_placeholder)
- #_________________________________________________________________________________________
+ # _________________________________________________________________________
- # task_permutations
+ # _collate_setup
- #_________________________________________________________________________________________
- def task_permutations(self, orig_args):
+ # _________________________________________________________________________
+ def _collate_setup(self):
"""
- k-permutations of n
-
- k-length tuples, all possible orderings, no self vs self
+ Finish setting up collate
"""
- decorator_name = "@permutations"
- error_type = error_task_permutations
- combinatorics_type = t_combinatorics_type.COMBINATORICS_PERMUTATIONS
- self.task_combinatorics (orig_args, combinatorics_type, decorator_name, error_type)
-
- #_________________________________________________________________________________________
+ #
+ # replace function / function names with tasks
+ #
+ input_files_task_globs = self._handle_tasks_globs_in_inputs(self.parsed_args["input"],
+ t_extra_inputs.KEEP_INPUTS)
+ ancestral_tasks = set(input_files_task_globs.tasks)
- # task_combinations
+ # how to transform input to output file name
+ file_names_transform = self._choose_file_names_transform(self.parsed_args,
+ (regex, formatter))
- #_________________________________________________________________________________________
- def task_combinations(self, orig_args):
- """
- k-length tuples
- Single (sorted) ordering, i.e. AB is the same as BA,
- No repeats. No AA, BB
+ modify_inputs = self.parsed_args["modify_inputs"]
+ if modify_inputs is not None:
+ modify_inputs = self._handle_tasks_globs_in_inputs(
+ modify_inputs, self.parsed_args["modify_inputs_mode"])
+ ancestral_tasks = ancestral_tasks.union(modify_inputs.tasks)
- E.g.
- combinations("ABCD", 3) = ['ABC', 'ABD', 'ACD', 'BCD']
- combinations("ABCD", 2) = ['AB', 'AC', 'AD', 'BC', 'BD', 'CD']
- """
- decorator_name = "@combinations"
- error_type = error_task_combinations
- combinatorics_type = t_combinatorics_type.COMBINATORICS_COMBINATIONS
- self.task_combinatorics (orig_args, combinatorics_type, decorator_name, error_type)
+ self.param_generator_func = collate_param_factory(input_files_task_globs,
+ # False, #
+ # flatten input
+ # removed
+ file_names_transform,
+ modify_inputs,
+ self.parsed_args["modify_inputs_mode"],
+ self.parsed_args["output"],
+ *self.parsed_args["extras"])
+ return ancestral_tasks
- #_________________________________________________________________________________________
+ # ========================================================================
- # task_combinations_with_replacement
+ # _decorator_mkdir
- #_________________________________________________________________________________________
- def task_combinations_with_replacement(self, orig_args):
+ # ========================================================================
+ def _decorator_mkdir(self, *unnamed_args, **named_args):
"""
- k-length tuples
- Single (sorted) ordering, i.e. AB is the same as BA,
- Repeats. AA, BB, AAC etc.
-
- E.g.
- combinations_with_replacement("ABCD", 3) = ['AAA', 'AAB', 'AAC', 'AAD',
- 'ABB', 'ABC', 'ABD',
- 'ACC', 'ACD',
- 'ADD',
- 'BBB', 'BBC', 'BBD',
- 'BCC', 'BCD',
- 'BDD',
- 'CCC', 'CCD',
- 'CDD',
- 'DDD']
- combinations_with_replacement("ABCD", 2) = ['AA', 'AB', 'AC', 'AD',
- 'BB', 'BC', 'BD',
- 'CC', 'CD',
- 'DD']
-
+ @mkdir
"""
- decorator_name = "@combinations_with_replacement"
- error_type = error_task_combinations_with_replacement
- combinatorics_type = t_combinatorics_type.COMBINATORICS_COMBINATIONS_WITH_REPLACEMENT
- self.task_combinatorics (orig_args, combinatorics_type, decorator_name, error_type)
+ syntax = "@mkdir"
+ description_with_args_placeholder = "%s(%%s)\n%s" % (
+ self.syntax, (self.description_with_args_placeholder % "..."))
+ self._prepare_preceding_mkdir(unnamed_args, named_args, syntax,
+ description_with_args_placeholder)
+ # _________________________________________________________________________
+ # mkdir
-
- #_________________________________________________________________________________________
-
- # task_transform
-
- #_________________________________________________________________________________________
- def task_transform (self, orig_args):
+ # _________________________________________________________________________
+ def mkdir(self, *unnamed_args, **named_args):
"""
- Merges multiple input files into a single output.
+ Make missing directories, including intermediates, before this task
"""
- decorator_name = "@transform"
- error_type = error_task_transform
- if len(orig_args) < 3:
- raise error_type(self, "Too few arguments for %s" % decorator_name)
+ syntax = "Task(name = %s).mkdir" % self._name
+ description_with_args_placeholder = "%s(%%s)" % (self.syntax)
+ self._prepare_preceding_mkdir(unnamed_args, named_args, syntax,
+ description_with_args_placeholder)
+ return self
+ # _________________________________________________________________________
- self.set_action_type (_task.action_task_transform)
+ # _prepare_dependent_mkdir
+ # _________________________________________________________________________
+ def _prepare_preceding_mkdir(self, unnamed_args, named_args, syntax,
+ task_description, defer = True):
+ """
+ Add mkdir Task to run before self
+ Common to
+ Task.mkdir
+ @mkdir
+ @follows(..., mkdir())
+ """
#
- # replace function / function names with tasks
+ # Create a new Task with a unique name to this instance of mkdir
#
- input_files_task_globs = self.handle_tasks_globs_in_inputs(orig_args[0])
+ self.cnt_task_mkdir += 1
+ cnt_task_mkdir_str = (" #%d" % self.cnt_task_mkdir) if self.cnt_task_mkdir > 1 else ""
+ task_name = r"mkdir%r%s before %s " % (unnamed_args, cnt_task_mkdir_str, self._name)
+ task_name = task_name.replace(",)", ")").replace(",", ", ")
+ new_task = self.pipeline._create_task(task_func=job_wrapper_mkdir, task_name=task_name)
+ # defer _add_parent so we can clone unless we are already
+ # calling add_parent (from _connect_parents())
+ if defer:
+ self.deferred_follow_params.append([task_description, False, [new_task]])
- #_________________________________________________________________________________
- #
- # single_job_single_output is bad policy. Can we remove it?
- # What does this actually mean in Ruffus semantics?
#
+ # Prepare new node
#
- # allows transform to take a single file or task
- if input_files_task_globs.single_file_to_list():
- self._single_job_single_output = self.single_job_single_output
+ new_task.syntax = syntax
+ new_task._prepare_mkdir(unnamed_args, named_args, task_description)
#
- # whether transform generates a list of jobs or not will depend on the parent task
+ # Hack:
+ # If the task name is too ugly,
+ # we can override it for flowchart printing using the
+ # display_name
#
- elif isinstance(input_files_task_globs.params, _task):
- self._single_job_single_output = input_files_task_globs.params
+ # new_node.display_name = ??? new_node.func_description
+ return new_task
- #_________________________________________________________________________________
+ # _________________________________________________________________________
- # how to transform input to output file name
- file_names_transform = self.choose_file_names_transform (orig_args[1], error_task_transform, decorator_name)
+ # _prepare_mkdir
+
+ # _________________________________________________________________________
+ def _prepare_mkdir(self, unnamed_args, named_args, task_description):
- orig_args = orig_args[2:]
+ self.error_type = error_task_mkdir
+ self._set_action_type(Task._action_mkdir)
+ self.needs_update_func = self.needs_update_func or needs_update_check_directory_missing
+ self.job_wrapper = job_wrapper_mkdir
+ self.job_descriptor = mkdir_job_descriptor
+ # doesn't have a real function
+ # use job_wrapper just so it is not None
+ self.user_defined_work_func = self.job_wrapper
#
- # inputs can also be defined by pattern match
+ # @transform like behaviour with regex / suffix or formatter
#
- extra_inputs, replace_inputs, output_pattern, extra_params = self.get_extra_inputs_outputs_extra (orig_args, error_type, decorator_name)
-
- self.param_generator_func = transform_param_factory ( input_files_task_globs,
- False, # flatten input
- file_names_transform,
- extra_inputs,
- replace_inputs,
- output_pattern,
- *extra_params)
- self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
- self.job_wrapper = job_wrapper_io_files
- self.job_descriptor = io_files_job_descriptor
- self.single_multi_io = self.many_to_many
-
- #_________________________________________________________________________________________
-
- # task_collate
-
- #_________________________________________________________________________________________
- def task_collate (self, orig_args):
- """
- Merges multiple input files into a single output.
- """
- decorator_name = "@collate"
- error_type = error_task_collate
- if len(orig_args) < 3:
- raise error_type(self, "Too few arguments for %s" % decorator_name)
+ if (len(unnamed_args) > 1 and
+ isinstance(unnamed_args[1], (formatter, suffix, regex))) or "filter" in named_args:
+ self.single_multi_io = self._many_to_many
+ self._setup_task_func = Task._transform_setup
- self.set_action_type (_task.action_task_collate)
+ #
+ # Parse named and unnamed arguments
+ #
+ self.parsed_args = parse_task_arguments(unnamed_args, named_args,
+ ["input", "filter", "modify_inputs",
+ "output", "output_dir", "extras"], task_description)
#
- # replace function / function names with tasks
+ # simple behaviour: just make directories in list of strings
#
- input_files_task_globs = self.handle_tasks_globs_in_inputs(orig_args[0])
+ # the mkdir decorator accepts one string, multiple strings or a list of strings
+ else:
+ #
+ # override funct description normally parsed from func.__doc__
+ # "Make missing directories including any intermediate
+ # directories on the specified path(s)"
+ #
+ self.func_description = "Make missing directories %s" % (
+ shorten_filenames_encoder(unnamed_args, 0))
- # how to transform input to output file name
- file_names_transform = self.choose_file_names_transform (orig_args[1], error_task_collate, decorator_name, (regex, formatter))
+ self.single_multi_io = self._one_to_one
+ self._setup_task_func = Task._do_nothing_setup
+ self.has_input_param = False
- orig_args = orig_args[2:]
+ #
+ #
+ #
+ # if a single argument collection of parameters, keep that as is
+ if len(unnamed_args) == 0:
+ self.parsed_args["output"] = []
+ elif len(unnamed_args) > 1:
+ self.parsed_args["output"] = unnamed_args
+ # len(unnamed_args) == 1: unpack unnamed_args[0]
+ elif non_str_sequence(unnamed_args[0]):
+ self.parsed_args["output"] = unnamed_args[0]
+ # single string or other non collection types
+ else:
+ self.parsed_args["output"] = unnamed_args
- #
- # inputs also defined by pattern match
- #
- extra_inputs, replace_inputs, output_pattern, extra_params = self.get_extra_inputs_outputs_extra (orig_args, error_type, decorator_name)
+ # all directories created in one job to reduce race conditions
+ # so we are converting [a,b,c] into [ [(a, b,c)] ]
+ # where unnamed_args = (a,b,c)
+ # i.e. one job whose solitory argument is a tuple/list of directory
+ # names
+ self.param_generator_func = args_param_factory([[sorted(self.parsed_args["output"], key = lambda x: str(x))]])
- self.single_multi_io = self.many_to_many
+ # print ("mkdir %s" % (self.func_description), file = sys.stderr)
- self.param_generator_func = collate_param_factory ( input_files_task_globs,
- False, # flatten input
- file_names_transform,
- extra_inputs,
- replace_inputs,
- output_pattern,
- *extra_params)
- self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
- self.job_wrapper = job_wrapper_io_files
- self.job_descriptor = io_files_job_descriptor
+ # ========================================================================
+ # _decorator_product
+ # ========================================================================
+ def _decorator_product(self, *unnamed_args, **named_args):
+ """
+ @product
+ """
+ self.syntax = "@product"
+ self.description_with_args_placeholder = "%s(%%s)\n%s" % (self.syntax,
+ self._get_decorated_function())
+ self._prepare_product(unnamed_args, named_args)
- #_________________________________________________________________________________________
+ # _________________________________________________________________________
- # task_merge
+ # _prepare_product
- #_________________________________________________________________________________________
- def task_merge (self, orig_args):
+ # _________________________________________________________________________
+ def _prepare_product(self, unnamed_args, named_args):
"""
- Merges multiple input files into a single output.
+ Common code for @product and pipeline.product
"""
+ self.error_type = error_task_product
+ self._set_action_type(Task._action_task_product)
+ self._setup_task_func = Task._product_setup
+ self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
+ self.job_wrapper = job_wrapper_io_files
+ self.job_descriptor = io_files_job_descriptor
+ self.single_multi_io = self._many_to_many
+
#
- # check enough arguments
+ # Parse named and unnamed arguments
#
- if len(orig_args) < 2:
- raise error_task_merge(self, "Too few arguments for @merge")
+ self.parsed_args = parse_task_arguments(unnamed_args, named_args,
+ ["input", "filter", "inputN", "modify_inputs",
+ "output", "extras"],
+ self.description_with_args_placeholder)
+
+ # _________________________________________________________________________
- self.set_action_type (_task.action_task_merge)
+ # _product_setup
+ # _________________________________________________________________________
+ def _product_setup(self):
+ """
+ Finish setting up product
+ """
#
# replace function / function names with tasks
#
- input_files_task_globs = self.handle_tasks_globs_in_inputs(orig_args[0])
-
- extra_params = orig_args[1:]
- self.param_generator_func = merge_param_factory (input_files_task_globs,
- *extra_params)
+ list_input_files_task_globs = [self._handle_tasks_globs_in_inputs(ii,
+ t_extra_inputs.KEEP_INPUTS)
+ for ii in self.parsed_args["input"]]
+ ancestral_tasks = set()
+ for input_files_task_globs in list_input_files_task_globs:
+ ancestral_tasks = ancestral_tasks.union(input_files_task_globs.tasks)
+ # how to transform input to output file name
+ file_names_transform = t_nested_formatter_file_names_transform(self,
+ self.parsed_args["filter"],
+ self.error_type,
+ self.syntax)
+
+ modify_inputs = self.parsed_args["modify_inputs"]
+ if modify_inputs is not None:
+ modify_inputs = self._handle_tasks_globs_in_inputs(
+ modify_inputs, self.parsed_args["modify_inputs_mode"])
+ ancestral_tasks = ancestral_tasks.union(modify_inputs.tasks)
+
+ self.param_generator_func = product_param_factory(list_input_files_task_globs,
+ # False, #
+ # flatten input
+ # removed
+ file_names_transform,
+ modify_inputs,
+ self.parsed_args["modify_inputs_mode"],
+ self.parsed_args["output"],
+ *self.parsed_args["extras"])
+
+ return ancestral_tasks
+
+ # ========================================================================
+
+ # _decorator_permutations
+ # _decorator_combinations
+ # _decorator_combinations_with_replacement
+
+ # ========================================================================
+ def _decorator_permutations(self, *unnamed_args, **named_args):
+ """
+ @permutations
+ """
+ self.syntax = "@permutations"
+ self.description_with_args_placeholder = "%s(%%s)\n%s" % (self.syntax,
+ self._get_decorated_function())
+ self._prepare_combinatorics(unnamed_args, named_args, error_task_permutations)
-# self._single_job_single_output = self.multiple_jobs_outputs
- self._single_job_single_output = self.single_job_single_output
- self.single_multi_io = self.many_to_one
+ def _decorator_combinations(self, *unnamed_args, **named_args):
+ """
+ @combinations
+ """
+ self.syntax = "@combinations"
+ self.description_with_args_placeholder = "%s(%%s)\n%s" % (self.syntax,
+ self._get_decorated_function())
+ self._prepare_combinatorics(unnamed_args, named_args, error_task_combinations)
- self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
- self.job_wrapper = job_wrapper_io_files
- self.job_descriptor = io_files_job_descriptor
+ def _decorator_combinations_with_replacement(self, *unnamed_args,
+ **named_args):
+ """
+ @combinations_with_replacement
+ """
+ self.syntax = "@combinations_with_replacement"
+ self.description_with_args_placeholder = "%s(%%s)\n%s" % (self.syntax,
+ self._get_decorated_function())
+ self._prepare_combinatorics(unnamed_args, named_args,
+ error_task_combinations_with_replacement)
- #_________________________________________________________________________________________
+ # _________________________________________________________________________
- # task_parallel
+ # _prepare_combinatorics
- #_________________________________________________________________________________________
- def task_parallel (self, orig_args):
+ # _________________________________________________________________________
+ def _prepare_combinatorics(self, unnamed_args, named_args, error_type):
"""
- calls user function in parallel
- with either each of a list of parameters
- or using parameters generated by a custom function
+ Common code for
+ @permutations and pipeline.permutations
+ @combinations and pipeline.combinations
+ @combinations_with_replacement and
+ pipeline.combinations_with_replacement
"""
- self.set_action_type (_task.action_parallel)
+ self.error_type = error_type
+ self._setup_task_func = Task._combinatorics_setup
+ self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
+ self.job_wrapper = job_wrapper_io_files
+ self.job_descriptor = io_files_job_descriptor
+ self.single_multi_io = self._many_to_many
- # unmodified from __init__
#
- # self.needs_update_func = None
- # self.job_wrapper = job_wrapper_generic
- # self.job_descriptor = io_files_job_descriptor
-
- if len(orig_args) == 0:
- raise error_task_parallel(self, "Too few arguments for @parallel")
-
- # Use parameters generated by a custom function
- if len(orig_args) == 1 and isinstance(orig_args[0], collections.Callable):
- #if len(orig_args) == 1 and type(orig_args[0]) == types.FunctionType:
- self.param_generator_func = args_param_factory(orig_args[0]())
-
- # list of params
- else:
- if len(orig_args) > 1:
- # single jobs
- params = copy.copy([orig_args])
- self._single_job_single_output = self.single_job_single_output
- else:
- # multiple jobs with input/output parameters etc.
- params = copy.copy(orig_args[0])
- check_parallel_parameters (self, params, error_task_parallel)
+ # Parse named and unnamed arguments
+ #
+ self.parsed_args = parse_task_arguments(unnamed_args, named_args,
+ ["input", "filter", "tuple_size",
+ "modify_inputs", "output", "extras"],
+ self.description_with_args_placeholder)
- self.param_generator_func = args_param_factory (params)
+ # _________________________________________________________________________
+ # _combinatorics_setup
+ # _________________________________________________________________________
+ def _combinatorics_setup(self):
+ """
+ Finish setting up combinatorics
+ """
+ #
+ # replace function / function names with tasks
+ #
+ input_files_task_globs = self._handle_tasks_globs_in_inputs(self.parsed_args["input"],
+ t_extra_inputs.KEEP_INPUTS)
+ ancestral_tasks = set(input_files_task_globs.tasks)
+
+ # how to transform input to output file name: len(k-tuples) of
+ # (identical) formatters
+ file_names_transform = t_nested_formatter_file_names_transform(
+ self, [self.parsed_args["filter"]] * self.parsed_args["tuple_size"],
+ self.error_type, self.syntax)
+
+ modify_inputs = self.parsed_args["modify_inputs"]
+ if modify_inputs is not None:
+ modify_inputs = self._handle_tasks_globs_in_inputs(
+ modify_inputs, self.parsed_args["modify_inputs_mode"])
+ ancestral_tasks = ancestral_tasks.union(modify_inputs.tasks)
+
+ # we are not going to specify what type of combinatorics this is twice:
+ # just look up from our error type
+ error_type_to_combinatorics_type = {
+ error_task_combinations_with_replacement:
+ t_combinatorics_type.COMBINATORICS_COMBINATIONS_WITH_REPLACEMENT,
+ error_task_combinations:
+ t_combinatorics_type.COMBINATORICS_COMBINATIONS,
+ error_task_permutations:
+ t_combinatorics_type.COMBINATORICS_PERMUTATIONS
+ }
+
+ self.param_generator_func = \
+ combinatorics_param_factory(input_files_task_globs,
+ # False, #
+ # flatten
+ # input
+ # removed
+ error_type_to_combinatorics_type[
+ self.error_type],
+ self.parsed_args["tuple_size"],
+ file_names_transform,
+ modify_inputs,
+ self.parsed_args["modify_inputs_mode"],
+ self.parsed_args["output"],
+ *self.parsed_args["extras"])
+
+ return ancestral_tasks
+
+ # ========================================================================
+
+ # _decorator_files
+
+ # ========================================================================
+ def _decorator_files(self, *unnamed_args, **named_args):
+ """
+ @files
+ """
+ self.syntax = "@files"
+ self.description_with_args_placeholder = "%s(%%s)\n%s" % (self.syntax,
+ self._get_decorated_function())
+ self._prepare_files(unnamed_args, named_args)
- #_________________________________________________________________________________________
+ # _________________________________________________________________________
- # task_files
+ # _prepare_files
- #_________________________________________________________________________________________
- def task_files (self, orig_args):
+ # _________________________________________________________________________
+ def _prepare_files(self, unnamed_args, named_args):
"""
- calls user function in parallel
- with either each of a list of parameters
- or using parameters generated by a custom function
-
- In the parameter list,
- The first two items of each set of parameters must
- be input/output files or lists of files or Null
+ Common code for @files and pipeline.files
"""
+ self.error_type = error_task_files
+ self._setup_task_func = Task._do_nothing_setup
+ self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
+ self.job_wrapper = job_wrapper_io_files
+ self.job_descriptor = io_files_job_descriptor
- self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
- self.job_wrapper = job_wrapper_io_files
- self.job_descriptor = io_files_job_descriptor
-
- if len(orig_args) == 0:
+ if len(unnamed_args) == 0:
raise error_task_files(self, "Too few arguments for @files")
# Use parameters generated by a custom function
- if len(orig_args) == 1 and isinstance(orig_args[0], collections.Callable):
- #if len(orig_args) == 1 and type(orig_args[0]) == types.FunctionType:
+ if len(unnamed_args) == 1 and isinstance(unnamed_args[0],
+ collections.Callable):
- self.set_action_type (_task.action_task_files_func)
- self.param_generator_func = files_custom_generator_param_factory(orig_args[0])
+ self._set_action_type(Task._action_task_files_func)
+ self.param_generator_func = files_custom_generator_param_factory(unnamed_args[0])
# assume
- self.single_multi_io = self.many_to_many
+ self.single_multi_io = self._many_to_many
# Use parameters in supplied list
else:
- self.set_action_type (_task.action_task_files)
+ self._set_action_type(Task._action_task_files)
- if len(orig_args) > 1:
+ if len(unnamed_args) > 1:
# single jobs
# This is true even if the previous task has multiple output
# These will all be joined together at the hip (like @merge)
# If you want different behavior, use @transform
- params = copy.copy([orig_args])
- self._single_job_single_output = self.single_job_single_output
- self.single_multi_io = self.one_to_one
-
+ params = copy.copy([unnamed_args])
+ self._is_single_job_single_output = self._single_job_single_output
+ self.single_multi_io = self._one_to_one
else:
# multiple jobs with input/output parameters etc.
- params = copy.copy(orig_args[0])
- self._single_job_single_output = self.multiple_jobs_outputs
- self.single_multi_io = self.many_to_many
-
- check_files_io_parameters (self, params, error_task_files)
-
- #
- # get list of function/function names and globs for all job params
- #
-
- #
- # replace function / function names with tasks
- #
- input_patterns = [j[0] for j in params]
- input_files_task_globs = self.handle_tasks_globs_in_inputs(input_patterns)
-
-
- #
- # extra params
- #
- output_extra_params = [tuple(j[1:]) for j in params]
-
- self.param_generator_func = files_param_factory (input_files_task_globs,
- False, # flatten input
- True, # do_not_expand_single_job_tasks
- output_extra_params)
+ params = copy.copy(unnamed_args[0])
+ self._is_single_job_single_output = self._multiple_jobs_outputs
+ self.single_multi_io = self._many_to_many
+ check_files_io_parameters(self, params, error_task_files)
+ self.parsed_args["input"] = [pp[0] for pp in params]
+ self.parsed_args["output"] = [tuple(pp[1:]) for pp in params]
+ self._setup_task_func = Task._files_setup
- #_________________________________________________________________________________________
+ # _________________________________________________________________________
- # task_files_re
+ # _files_setup
- #_________________________________________________________________________________________
- def task_files_re (self, old_args):
+ # _________________________________________________________________________
+ def _files_setup(self):
"""
- calls user function in parallel
- with input_files, output_files, parameters
- These needed to be generated on the fly by
- getting all file names in the supplied list/glob pattern
- There are two variations:
-
- 1) inputfiles = all files in glob which match the regular expression
- outputfile = generated from the replacement string
-
- 2) inputfiles = all files in glob which match the regular expression and
- generated from the "from" replacement string
- outputfiles = all files in glob which match the regular expression and
- generated from the "to" replacement string
+ Finish setting up @files
"""
#
- # check enough arguments
+ # replace function / function names with tasks
#
- if len(old_args) < 3:
- raise error_task_files_re(self, "Too few arguments for @files_re")
+ input_files_task_globs = self._handle_tasks_globs_in_inputs(self.parsed_args["input"],
+ t_extra_inputs.KEEP_INPUTS)
- self.set_action_type (_task.action_task_files_re)
+ self.param_generator_func = files_param_factory(input_files_task_globs,
+ True,
+ self.parsed_args["output"])
+ return set(input_files_task_globs.tasks)
- # check if parameters wrapped in combine
- combining_all_jobs, orig_args = is_file_re_combining(old_args)
+ # ========================================================================
- #
- # replace function / function names with tasks
- #
- input_files_task_globs = self.handle_tasks_globs_in_inputs(orig_args[0])
+ # _decorator_parallel
+
+ # ========================================================================
+ def _decorator_parallel(self, *unnamed_args, **named_args):
+ """
+ @parallel
+ """
+ self.syntax = "@parallel"
+ self._prepare_parallel(unnamed_args, named_args)
- file_names_transform = t_regex_file_names_transform(self, regex(orig_args[1]), error_task_files_re, "@files_re")
+ # _________________________________________________________________________
+ # _prepare_parallel
- # if the input file term is missing, just use the original
- if len(orig_args) == 3:
- extra_input_files_task_globs = None
- output_and_extras = [orig_args[2]]
- else:
- extra_input_files_task_globs = self.handle_tasks_globs_in_inputs(orig_args[2])
- output_and_extras = orig_args[3:]
+ # _________________________________________________________________________
+ def _prepare_parallel(self, unnamed_args, named_args):
+ """
+ Common code for @parallel and pipeline.parallel
+ """
+ self.error_type = error_task_parallel
+ self._set_action_type(Task._action_task_parallel)
+ self._setup_task_func = Task._do_nothing_setup
+ self.needs_update_func = None
+ self.job_wrapper = job_wrapper_generic
+ self.job_descriptor = io_files_job_descriptor
+
+ if len(unnamed_args) == 0:
+ raise error_task_parallel(self, "Too few arguments for @parallel")
+ # Use parameters generated by a custom function
+ if len(unnamed_args) == 1 and isinstance(unnamed_args[0],
+ collections.Callable):
+ self.param_generator_func = args_param_factory(unnamed_args[0]())
- if combining_all_jobs:
- self.single_multi_io = self.many_to_many
- self.param_generator_func = collate_param_factory (input_files_task_globs,
- False, # flatten
- file_names_transform,
- extra_input_files_task_globs,
- t_extra_inputs.REPLACE_INPUTS,
- *output_and_extras)
+ # list of params
else:
+ if len(unnamed_args) > 1:
+ # single jobs
+ params = copy.copy([unnamed_args])
+ self._is_single_job_single_output = self._single_job_single_output
+ else:
+ # multiple jobs with input/output parameters etc.
+ params = copy.copy(unnamed_args[0])
+ check_parallel_parameters(self, params, error_task_parallel)
- self.single_multi_io = self.many_to_many
- self.param_generator_func = transform_param_factory (input_files_task_globs,
- False, # flatten
- file_names_transform,
- extra_input_files_task_globs,
- t_extra_inputs.REPLACE_INPUTS,
- *output_and_extras)
+ self.param_generator_func = args_param_factory(params)
+ # ========================================================================
- self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
- self.job_wrapper = job_wrapper_io_files
- self.job_descriptor = io_files_job_descriptor
+ # _decorator_files_re
+ # ========================================================================
+ def _decorator_files_re(self, *unnamed_args, **named_args):
+ """
+ @files_re
+ calls user function in parallel
+ with input_files, output_files, parameters
+ These needed to be generated on the fly by
+ getting all file names in the supplied list/glob pattern
+ There are two variations:
- #_________________________________________________________________________________________
+ 1) inputfiles = all files in glob which match the regular
+ expression
+ outputfile = generated from the replacement string
- # task_mkdir
+ 2) inputfiles = all files in glob which match the regular
+ expression and generated from the "from"
+ replacement string
+ outputfiles = all files in glob which match the regular
+ expression and generated from the "to"
+ replacement string
+ """
+ self.syntax = "@files_re"
+ self.error_type = error_task_files_re
+ self._set_action_type(Task._action_task_files_re)
+ self.needs_update_func = self.needs_update_func or needs_update_check_modify_time
+ self.job_wrapper = job_wrapper_io_files
+ self.job_descriptor = io_files_job_descriptor
+ self.single_multi_io = self._many_to_many
- # only called within task_follows
+ if len(unnamed_args) < 3:
+ raise self.error_type(self, "Too few arguments for @files_re")
- #_________________________________________________________________________________________
- def task_mkdir (self, orig_args):
- self.cnt_task_mkdir += 1
- # give unique name to this instance of mkdir
- unique_name = r"(mkdir %d) before " % self.cnt_task_mkdir + self._name
- new_node = _task(self._module_name, unique_name)
- self.add_child(new_node)
- new_node.do_task_mkdir(orig_args)
- new_node.display_name = new_node._description
+ # 888888888888888888888888888888888888888888888888888888888888888888888
+ # !! HERE BE DRAGONS !!
- def do_task_mkdir (self, orig_args):
- """
- list of directory names or a single argument which is aa list of directory names
- Creates directory if missing
- """
- decorator_name = "mkdir"
- error_type = error_task_mkdir
+ # Legacy, deprecated parameter handling depending on positions
+ # and not even on type
- # jump through hoops
- self.set_action_type (_task.action_mkdir)
- self.needs_update_func = self.needs_update_func or needs_update_check_directory_missing
- # don't shorten in description: full path
- self._description = "Make directories %s" % (shorten_filenames_encoder(orig_args, 0))
- self.job_wrapper = job_wrapper_mkdir
- self.job_descriptor = mkdir_job_descriptor
+ # check if parameters wrapped in combine
+ combining_all_jobs, unnamed_args = is_file_re_combining(unnamed_args)
- # doesn't have a real function
- # use job_wrapper just so it is not None
- self.user_defined_work_func = self.job_wrapper
+ # second parameter is always regex()
+ unnamed_args[1] = regex(unnamed_args[1])
+ # third parameter is inputs() if there is a four and fifth parameter...
+ # That means if you want "extra" parameters, you always need inputs()
+ if len(unnamed_args) > 3:
+ unnamed_args[2] = inputs(unnamed_args[2])
- #
- # @transform like behaviour with regex / suffix or formatter
- #
- if len(orig_args) > 1 and isinstance(orig_args[1], (formatter, suffix, regex)):
- self.single_multi_io = self.many_to_many
+ # 888888888888888888888888888888888888888888888888888888888888888888888
- if len(orig_args) < 3:
- raise error_type(self, "Too few arguments for %s" % decorator_name)
+ self.description_with_args_placeholder = "%s(%%s)\n%s" % (self.syntax,
+ self._get_decorated_function())
+ self.parsed_args = parse_task_arguments(unnamed_args, named_args,
+ ["input", "filter", "modify_inputs",
+ "output", "extras"],
+ self.description_with_args_placeholder)
- #
- # replace function / function names with tasks
- #
- input_files_task_globs = self.handle_tasks_globs_in_inputs(orig_args[0])
+ if combining_all_jobs:
+ self._setup_task_func = Task._collate_setup
+ else:
+ self._setup_task_func = Task._transform_setup
+ # 8888888888888888888888888888888888888888888888888888888888888888888888888
- # how to transform input to output file name
- file_names_transform = self.choose_file_names_transform (orig_args[1], error_task_transform, decorator_name)
+ # Task functions
- orig_args = orig_args[2:]
+ # follows
+ # check_if_uptodate
+ # posttask
+ # jobs_limit
+ # active_if
+ # graphviz
- #
- # inputs can also be defined by pattern match
- #
- extra_inputs, replace_inputs, output_pattern, extra_params = self.get_extra_inputs_outputs_extra (orig_args, error_type, decorator_name)
+ # 8888888888888888888888888888888888888888888888888888888888888888888888888
- if len(extra_params):
- raise error_type(self, "Too many arguments for %s" % decorator_name)
+ # ========================================================================
+ # follows
- self.param_generator_func = transform_param_factory ( input_files_task_globs,
- False, # flatten input
- file_names_transform,
- extra_inputs,
- replace_inputs,
- output_pattern,
- *extra_params)
+ # ========================================================================
+ def follows(self, *unnamed_args, **named_args):
+ """
+ Specifies a preceding task / action which this task will follow.
+ The preceding task can be specified as a string or function or Task
+ object.
+ A task can also follow the making of one or more directories:
- #
- # simple behaviour: just make directories in list of strings
- #
- # the mkdir decorator accepts one string, multiple strings or a list of strings
- else:
- self.single_multi_io = self.one_to_one
+ task.follows(mkdir("my_dir"))
- #
- #
- #
- # if a single argument collection of parameters, keep that as is
- if len(orig_args) == 0:
- mkdir_params = []
- elif len(orig_args) > 1:
- mkdir_params = orig_args
- # len(orig_args) == 1: unpack orig_args[0]
- elif non_str_sequence (orig_args[0]):
- mkdir_params = orig_args[0]
- # single string or other non collection types
- else:
- mkdir_params = orig_args
+ """
+ description_with_args_placeholder = (
+ self.description_with_args_placeholder % "...") + ".follows(%r)"
- # all directories created in one job to reduce race conditions
- # so we are converting [a,b,c] into [ [(a, b,c)] ]
- # where orig_args = (a,b,c)
- # i.e. one job whose solitory argument is a tuple/list of directory names
- self.param_generator_func = args_param_factory([[sorted(mkdir_params)]])
+ self.deferred_follow_params.append([description_with_args_placeholder, False,
+ unnamed_args])
+ #self._connect_parents(description_with_args_placeholder, False,
+ # unnamed_args)
+ return self
+
+ # _________________________________________________________________________
+ # _decorator_follows
+ # _________________________________________________________________________
+ def _decorator_follows(self, *unnamed_args, **named_args):
+ """
+ unnamed_args can be string or function or Task
+ For strings, if lookup fails, will defer.
+ """
+ description_with_args_placeholder = "@follows(%r)\n" + (
+ self.description_with_args_placeholder % "...")
+ self.deferred_follow_params.append([description_with_args_placeholder, False,
+ unnamed_args])
+ #self._connect_parents(description_with_args_placeholder, False, unnamed_args)
+ # _________________________________________________________________________
+ # _complete_setup
+ # _________________________________________________________________________
+ def _complete_setup(self):
+ """
+ Connect up parents if follows was specified and setups up task functions
+ Returns a set of parent tasks
- #8888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+ Note will tear down previous parental links before doing anything
+ """
+ #DEBUGGG
+ #print(" task._complete_setup start %s" % (self._get_display_name(), ), file = sys.stderr)
+ self._remove_all_parents()
+ ancestral_tasks = self._deferred_connect_parents()
+ ancestral_tasks |= self._setup_task_func(self)
+ #DEBUGGG
+ #print(" task._complete_setup finish %s\n" % (self._get_display_name(), ), file = sys.stderr)
+ return ancestral_tasks
- # Other task handlers
+ # _________________________________________________________________________
+ # _deferred_connect_parents
+ # _________________________________________________________________________
+ def _deferred_connect_parents(self):
+ """
+ Called by _complete_task_setup() from pipeline_run, pipeline_printout etc.
+ returns a non-redundant list of all the ancestral tasks
+ """
+ # DEBUGGG
+ #print(" task._deferred_connect_parents start %s (%d to do)" % (self._get_display_name(), len(self.deferred_follow_params)), file = sys.stderr)
+ parent_tasks = set()
- #8888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+ for ii, deferred_follow_params in enumerate(self.deferred_follow_params):
+ #DEBUGGG
+ #print(" task._deferred_connect_parents %s %d out of %d " % (self._get_display_name(), ii, len(self.deferred_follow_params)), file = sys.stderr)
+ new_tasks = self._connect_parents(*deferred_follow_params)
+ # convert to mkdir and dynamically created tasks from follows into the actual created tasks
+ # otherwise each time we redo this, we will have a sorceror's apprentice situation!
+ deferred_follow_params[2] = new_tasks
+ parent_tasks.update(new_tasks)
+ # DEBUGGG
+ #print(" task._deferred_connect_parents finish %s" % self._get_display_name(), file = sys.stderr)
+ return parent_tasks
- #_________________________________________________________________________________________
+ # _________________________________________________________________________
- # task_follows
+ # _connect_parents
+ # Deferred tasks will need to be resolved later
+ # Because deferred tasks can belong to other pipelines
- #_________________________________________________________________________________________
- def task_follows (self, args):
+ # _________________________________________________________________________
+ def _connect_parents(self, description_with_args_placeholder, no_mkdir,
+ unnamed_args):
"""
- Saved decorator arguments should be:
- (string/task,...)
+ unnamed_args can be string or function or Task
+ For strings, if lookup fails, will defer.
+
+ Called from
+ * task.follows
+ * @follows
+ * decorators, e.g. @transform _handle_tasks_globs_in_inputs
+ (input dependencies)
+ * pipeline.transform etc. _handle_tasks_globs_in_inputs
+ (input dependencies)
+ * @split / pipeline.split _handle_tasks_globs_in_inputs
+ (output dependencies)
"""
+ # DEBUGGG
+ #print(" _connect_parents start %s" % self._get_display_name(), file = sys.stderr)
new_tasks = []
- for arg in args:
+ for arg in unnamed_args:
#
- # specified by string: unicode or otherwise
+ # Task
#
- if isinstance(arg, path_str_type):
- # string looks up to defined task, use that
- if node.is_node(arg):
- arg = node.lookup_node_from_name(arg)
- # string looks up to defined task in main module, use that
- elif node.is_node("__main__." + arg):
- arg = node.lookup_node_from_name("__main__." + arg)
+ if isinstance(arg, Task):
+ if arg == self:
+ raise error_decorator_args(
+ "Cannot have a task as its own (circular) dependency:\n"
+ % description_with_args_placeholder % (arg,))
#
- # string does not look up to defined task: defer
+ # re-lookup from task name to handle cloning
#
+ if arg.pipeline.name == self.pipeline.original_name and \
+ self.pipeline.original_name != self.pipeline.name:
+ tasks = lookup_tasks_from_name(arg._name,
+ default_pipeline_name=self.pipeline.name,
+ default_module_name=self.func_module_name)
+ new_tasks.extend(tasks)
+
+ if not tasks:
+ raise error_node_not_task(
+ "task '%s' '%s::%s' is somehow absent in the cloned pipeline (%s)!%s"
+ % (self.pipeline.original_name, arg._name, self.pipeline.name,
+ description_with_args_placeholder % (arg._name,)))
else:
- # no module: use same module as current task
- names = arg.rsplit(".", 2)
- if len(names) == 1:
- arg = _task(self._module_name, arg)
- else:
- arg = _task(*names)
+ new_tasks.append(arg)
- #
- # add dependency
- # duplicate dependencies are ignore automatically
- #
- self.add_child(arg)
- new_tasks.append(arg)
+ #
+ # Pipeline: defer
+ #
+ elif isinstance(arg, Pipeline):
+ if arg == self.pipeline:
+ raise error_decorator_args("Cannot have your own pipeline as a (circular) "
+ "dependency of a Task:\n" +
+ description_with_args_placeholder % (arg,))
+
+ if not len(arg.get_tail_tasks()):
+ raise error_no_tail_tasks("Pipeline '{pipeline_name}' has no 'tail' tasks defined.\nWhich task "
+ "in '{pipeline_name}' are you referring to?"
+ .format(pipeline_name = arg.name))
+ new_tasks.extend(arg.get_tail_tasks())
+
+ #
+ # specified by string: unicode or otherwise
+ #
+ elif isinstance(arg, path_str_type):
+ # handle pipeline cloning
+ task_name = arg.replace(self.pipeline.original_name + "::",
+ self.pipeline.name + "::")
+ tasks = lookup_tasks_from_name(arg,
+ default_pipeline_name=self.pipeline.name,
+ default_module_name=self.func_module_name)
+ new_tasks.extend(tasks)
+
+ if not tasks:
+ raise error_node_not_task("task '%s' is not a pipelined task in Ruffus. "
+ "Have you mis-spelt the function or task name?\n%s"
+ % (arg, description_with_args_placeholder % (arg,)))
#
# for mkdir, automatically generate task with unique name
#
elif isinstance(arg, mkdir):
- self.cnt_task_mkdir += 1
- # give unique name to this instance of mkdir
- unique_name = r"(mkdir %d) before " % self.cnt_task_mkdir + self._name
- new_node = _task(self._module_name, unique_name)
- self.add_child(new_node)
- new_node.do_task_mkdir(arg.args)
- new_node.display_name = new_node._description
- new_tasks.append(new_node)
-
-
+ if no_mkdir:
+ raise error_decorator_args("Unexpected mkdir() found.\n" +
+ description_with_args_placeholder % (arg,))
+ # syntax for new task doing the mkdir
+ if self.created_via_decorator:
+ mkdir_task_syntax = "@follows(mkdir())"
+ else:
+ mkdir_task_syntax = "Task(name=%r).follows(mkdir())" % self._get_display_name()
+ mkdir_description_with_args_placeholder = \
+ description_with_args_placeholder % "mkdir(%s)"
+ new_tasks.append(self._prepare_preceding_mkdir(arg.args, {}, mkdir_task_syntax,
+ mkdir_description_with_args_placeholder, False))
#
# Is this a function?
# Turn this function into a task
# (add task as attribute of this function)
# Add self as dependent
- else:
- #if type(arg) != types.FunctionType:
- if not isinstance(arg, collections.Callable):
+ elif isinstance(arg, collections.Callable):
+ task = lookup_unique_task_from_func(arg, default_pipeline_name=self.pipeline.name)
- raise error_decorator_args("Dependencies must be functions or function names in " +
- "@task_follows %s:\n[%s]" %
- (self._name, str(arg)))
+ # add new task to pipeline if necessary
+ if not task:
+ task = main_pipeline._create_task(task_func=arg)
+ new_tasks.append(task)
+
+ else:
+ raise error_decorator_args("Expecting a function or function name or task name or "
+ "Task or Pipeline.\n" +
+ description_with_args_placeholder % (arg,))
- # add task as attribute of this function
- if not hasattr(arg, "pipeline_task"):
- arg.pipeline_task = _task.create_task(arg)
- self.add_child(arg.pipeline_task)
- new_tasks.append(arg.pipeline_task)
+ #
+ # add dependency
+ # duplicate dependencies are ignore automatically
+ #
+ for task in new_tasks:
+ self._add_parent(task)
+ # DEBUGGG
+ #print(" _connect_parents finish %s" % self._get_display_name(), file = sys.stderr)
return new_tasks
+ # ========================================================================
+ # check_if_uptodate
+
+ # ========================================================================
+ def check_if_uptodate(self, func):
+ """
+ Specifies how a task is to be checked if it needs to be rerun (i.e. is
+ up-to-date).
+ func returns true if input / output files are up to date
+ func takes as many arguments as the task function
+ """
+ if not isinstance(func, collections.Callable):
+ description_with_args_placeholder = \
+ (self.description_with_args_placeholder % "...") + ".check_if_uptodate(%r)"
+ raise error_decorator_args("Expected a single function or Callable object in \n" +
+ description_with_args_placeholder % (func,))
+ self.needs_update_func = func
+ return self
- #_________________________________________________________________________________________
+ # _________________________________________________________________________
- # task_check_if_uptodate
+ # _decorator_check_if_uptodate
- #_________________________________________________________________________________________
- def task_check_if_uptodate (self, args):
+ # _________________________________________________________________________
+ def _decorator_check_if_uptodate(self, *args):
"""
- Saved decorator arguments should be:
- a function which takes the appropriate number of arguments for each job
+ @check_if_uptodate
"""
if len(args) != 1 or not isinstance(args[0], collections.Callable):
- #if len(args) != 1 or type(args[0]) != types.FunctionType:
- raise error_decorator_args("Expecting a single function in " +
- "@task_check_if_uptodate %s:\n[%s]" %
- (self._name, str(args)))
- self.needs_update_func = args[0]
+ description_with_args_placeholder = "@check_if_uptodate(%r)\n" + (
+ self.description_with_args_placeholder % "...")
+ raise error_decorator_args("Expected a single function or Callable object in \n" +
+ description_with_args_placeholder % (args,))
+ self.needs_update_func = args[0]
+
+ # ========================================================================
+
+ # posttask
+
+ # ========================================================================
+ def posttask(self, *funcs):
+ """
+ Takes one or more functions which will be called if the task completes
+ """
+ description_with_args_placeholder = ("Expecting simple functions or touch_file() in \n" +
+ (self.description_with_args_placeholder % "...") +
+ ".posttask(%r)")
+ self._set_posttask(description_with_args_placeholder, *funcs)
+ return self
+ # _________________________________________________________________________
+ # _decorator_posttask
+
+ # _________________________________________________________________________
+ def _decorator_posttask(self, *funcs):
+ """
+ @posttask
+ """
+ description_with_args_placeholder = ("Expecting simple functions or touch_file() in \n" +
+ "@posttask(%r)\n" +
+ (self.description_with_args_placeholder % "..."))
+ self._set_posttask(description_with_args_placeholder, *funcs)
- #_________________________________________________________________________________________
+ # _________________________________________________________________________
- # task_posttask
+ # _set_posttask
- #_________________________________________________________________________________________
- def task_posttask(self, args):
+ # _________________________________________________________________________
+ def _set_posttask(self, description_with_args_placeholder, *funcs):
"""
- Saved decorator arguments should be:
- one or more functions which will be called if the task completes
+ Takes one or more functions which will be called if the task completes
"""
- for arg in args:
+ for arg in funcs:
if isinstance(arg, touch_file):
- self.posttask_functions.append(touch_file_factory (arg.args, register_cleanup))
+ self.posttask_functions.append(touch_file_factory(arg.args, register_cleanup))
elif isinstance(arg, collections.Callable):
- #elif type(arg) == types.FunctionType:
self.posttask_functions.append(arg)
else:
- raise PostTaskArgumentError("Expecting simple functions or touch_file in " +
- "@posttask(...)\n Task = %s" %
- (self._name))
+ raise PostTaskArgumentError(description_with_args_placeholder % (arg,))
+
+ # ========================================================================
+
+ # jobs_limit
+
+ # ========================================================================
+ def jobs_limit(self, maximum_jobs_in_parallel, limit_name=None):
+ """
+ Limit the number of concurrent jobs
+ """
+ description_with_args_placeholder = ((self.description_with_args_placeholder % "...") +
+ ".jobs_limit(%r%s)")
+ self._set_jobs_limit(description_with_args_placeholder,
+ maximum_jobs_in_parallel, limit_name)
+ return self
+
+ # _________________________________________________________________________
+
+ # _decorator_jobs_limit
+
+ # _________________________________________________________________________
+ def _decorator_jobs_limit(self, maximum_jobs_in_parallel, limit_name=None):
+ """
+ @jobs_limit
+ """
+ description_with_args_placeholder = ("@jobs_limit(%r%s)\n" +
+ (self.description_with_args_placeholder % "..."))
+ self._set_jobs_limit(description_with_args_placeholder,
+ maximum_jobs_in_parallel, limit_name)
- #_________________________________________________________________________________________
+ # _________________________________________________________________________
- # task_jobs_limit
+ # _set_jobs_limit
- #_________________________________________________________________________________________
- def task_jobs_limit(self, args):
- """
- Limit the number of concurrent jobs
- """
- maximum_jobs, name = (args + (None,))[0:2]
+ # _________________________________________________________________________
+ def _set_jobs_limit(self, description_with_args_placeholder,
+ maximum_jobs_in_parallel, limit_name=None):
try:
- maximum_jobs_num = int(maximum_jobs)
- assert(maximum_jobs_num >= 1)
+ maximum_jobs_in_parallel = int(maximum_jobs_in_parallel)
+ assert(maximum_jobs_in_parallel >= 1)
except:
- limit_name = ", " + name if name else ""
- raise JobsLimitArgumentError(('In @jobs_limit(%s%s), the limit '
- 'must be an integer number greater than or '
- 'equal to 1') %
- (maximum_jobs_num, limit_name))
- if name != None:
- self.semaphore_name = name
- if self.semaphore_name in self.job_limit_semaphores:
- curr_maximum_jobs = self.job_limit_semaphores[self.semaphore_name]
- if curr_maximum_jobs != maximum_jobs_num:
- raise JobsLimitArgumentError(('@jobs_limit(%d, "%s") cannot ' +
- 're-defined with a different limit of %d') %
- (self.semaphore_name, curr_maximum_jobs,
- maximum_jobs_num))
+ limit_name = ", " + limit_name if limit_name else ""
+ raise JobsLimitArgumentError("Expecting a positive integer > 1 in \n" +
+ description_with_args_placeholder
+ % (maximum_jobs_in_parallel, limit_name))
+
+ # set semaphore name to other than the "pipeline.name:task name"
+ if limit_name is not None:
+ self.semaphore_name = limit_name
+ if self.semaphore_name in self._job_limit_semaphores:
+ prev_maximum_jobs = self._job_limit_semaphores[self.semaphore_name]
+ if prev_maximum_jobs != maximum_jobs_in_parallel:
+ limit_name = ", " + limit_name if limit_name else ""
+ raise JobsLimitArgumentError('The job limit %r cannot re-defined from the former '
+ 'limit of %d in \n'
+ % (self.semaphore_name, prev_maximum_jobs) +
+ description_with_args_placeholder
+ % (maximum_jobs_in_parallel, limit_name))
else:
#
# save semaphore and limit
#
- self.job_limit_semaphores[self.semaphore_name] = maximum_jobs_num
-
+ self._job_limit_semaphores[
+ self.semaphore_name] = maximum_jobs_in_parallel
- #_________________________________________________________________________________________
+ # ========================================================================
- # task_active_if
+ # active_if
- #_________________________________________________________________________________________
- def task_active_if (self, active_if_checks):
+ # ========================================================================
+ def active_if(self, *active_if_checks):
"""
If any of active_checks is False or returns False, then the task is
marked as "inactive" and its outputs removed.
"""
- #print 'job is active:', active_checks, [
- # arg() if isinstance(arg, collections.Callable) else arg
- # for arg in active_checks]
- if self.active_if_checks == None:
+ # print 'job is active:', active_checks, [
+ # arg() if isinstance(arg, collections.Callable) else arg
+ # for arg in active_checks]
+ if self.active_if_checks is None:
self.active_if_checks = []
self.active_if_checks.extend(active_if_checks)
- #print(self.active_if_checks)
+ # print(self.active_if_checks)
+ return self
+
+ # _________________________________________________________________________
+
+ # _decorator_active_if
+
+ # _________________________________________________________________________
+ def _decorator_active_if(self, *active_if_checks):
+ """
+ @active_if
+ """
+ self.active_if(*active_if_checks)
+
+ # ========================================================================
+
+ # _decorator_graphviz
+
+ # ========================================================================
+ def graphviz(self, *unnamed_args, **named_args):
+ """
+ Sets graphviz (e.g. `dot`) attributes used to draw this Task
+ """
+ self.graphviz_attributes = named_args
+ if len(unnamed_args):
+ raise TypeError("Only named arguments expected in :" +
+ self.description_with_args_placeholder % "..." +
+ ".graphviz(%r)\n" % unnamed_args)
+ return self
+
+ # _________________________________________________________________________
+
+ # _decorator_graphviz
+ # _________________________________________________________________________
+ def _decorator_graphviz(self, *unnamed_args, **named_args):
+ self.graphviz_attributes = named_args
+ if len(unnamed_args):
+ raise TypeError("Only named arguments expected in :" +
+ "@graphviz(%r)\n" % unnamed_args +
+ self.description_with_args_placeholder % "...")
- #_________________________________________________________________________________________
+# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- # task_graphviz
+# End of Task
- #_________________________________________________________________________________________
- def task_graphviz(self, *unnamed_args, **named_args):
- self.graphviz_attributes=named_args
+# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
class task_encoder(json.JSONEncoder):
+
def default(self, obj):
if isinstance(obj, set):
return list(obj)
if isinstance(obj, defaultdict):
return dict(obj)
- if isinstance(obj, _task):
- return obj._name #, _task.action_names[obj.action_task], obj._description]
+ if isinstance(obj, Task):
+ # , Task._action_names[obj._action_task], obj.func_description]
+ return obj._name
return json.JSONEncoder.default(self, obj)
+# _____________________________________________________________________________
+# is_node_up_to_date
+# _____________________________________________________________________________
+def is_node_up_to_date(node, extra_data):
+ """
+ Forwards tree depth first search "signalling" mechanism to
+ node _is_up_to_date method
+ Depth first search stops when node._is_up_to_date return True
+ """
+ return node._is_up_to_date(extra_data)
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
# Functions
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-#_________________________________________________________________________________________
-
-# link_task_names_to_functions
-
-#_________________________________________________________________________________________
-def link_task_names_to_functions ():
- """
- Make sure all tasks in dependency list are linked to real functions
- Call this before running anything else
- """
-
- for n in node._all_nodes:
- if n.user_defined_work_func == None:
- dependent_display_task_name = n._inward[0].get_task_name()
- if n._module_name in sys.modules:
- module = sys.modules[n._module_name]
- if hasattr(module, n._func_name):
- n.user_defined_work_func = getattr(module, n._func_name)
- else:
- raise error_decorator_args(("Module '%s' has no function '%s' in " +
- "\n at task_follows('%s')\ndef %s...") %
- (n._module_name, n._func_name, n.get_task_name(), dependent_display_task_name))
- else:
- raise error_decorator_args("Module '%s' not found in " +
- "\n at task_follows('%s')\ndef %s..." %
- (n._module_name, n.get_task_name(), dependent_display_task_name))
-
+# 88888888888888888888888888888888888888888888888888888888888888888888888888888
- #
- # some jobs single state status mirrors parent's state
- # and parent task not known until know
- #
- if isinstance(n._single_job_single_output, _task):
- n._single_job_single_output = n._single_job_single_output._single_job_single_output
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
# update_checksum_level_on_tasks
-#_________________________________________________________________________________________
-def update_checksum_level_on_tasks (checksum_level):
+# _____________________________________________________________________________
+def update_checksum_level_on_tasks(checksum_level):
"""Reset the checksum level for all tasks"""
for n in node._all_nodes:
n.checksum_level = checksum_level
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
# update_active_states_for_all_tasks
-#_________________________________________________________________________________________
-def update_active_states_for_all_tasks ():
+# _____________________________________________________________________________
+def update_active_states_for_all_tasks():
"""
@active_if decorated tasks can change their active state every time
@@ -2782,110 +4228,99 @@ def update_active_states_for_all_tasks ():
"""
for n in node._all_nodes:
- n.update_active_state()
+ n._update_active_state()
+
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
-# task_names_to_tasks
+# lookup_pipeline
-#_________________________________________________________________________________________
-def task_names_to_tasks (task_description, task_names):
+# _____________________________________________________________________________
+def lookup_pipeline(pipeline):
"""
- Given a list of task names, look up the corresponding tasks
- Will just pass through if the task_name is already a task
+ If pipeline is
+ None : main_pipeline
+ string : lookup name in pipelines
"""
- #
- # In case we are given a single item instead of a list
- # accepts unicode
- #
- if isinstance(task_names, path_str_type) or isinstance(task_names, collections.Callable):
- #if isinstance(task_names, basestring) or type(task_names) == types.FunctionType:
- task_names = [task_names]
-
- task_nodes = []
- for task_name in task_names:
-
- # Is this already a function, don't do mapping if already is task
- if isinstance(task_name, collections.Callable):
- #if type(task_name) == types.FunctionType:
- if hasattr(task_name, "pipeline_task"):
- task_nodes.append(task_name.pipeline_task)
- continue
- else:
- # blow up for unwrapped function
- raise error_function_is_not_a_task(("Function def %s(...): is not a pipelined task in ruffus." %
- task_name.__name__) +
- " To include this, this function needs to have a ruffus "+
- "decoration like '@parallel', '@files', or named as a dependent "+
- "of some other Ruffus task function via '@follows'.")
-
- # assumes is some kind of string
- if not node.is_node(task_name):
- if node.is_node("__main__." + task_name):
- task_nodes.append(node.lookup_node_from_name("__main__." + task_name))
- else:
- raise error_node_not_task("%s task '%s' is not a pipelined task in Ruffus. Have you mis-spelt the function name?" % (
- task_description, task_name))
- else:
- task_nodes.append(node.lookup_node_from_name(task_name))
- return task_nodes
+ # Pipeline object pass through unchanged
+ if isinstance(pipeline, Pipeline):
+ return pipeline
+ # default to main_pipeline if None
+ if not pipeline:
+ return main_pipeline
+ # strings: lookup from name
+ if isinstance(pipeline, str) and pipeline in Pipeline.pipelines:
+ return Pipeline.pipelines[pipeline]
+ raise error_not_a_pipeline("%s does not name a pipeline." % pipeline)
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
# pipeline_printout_in_dot_format
-#_________________________________________________________________________________________
-def pipeline_printout_graph (stream,
- output_format = None,
- target_tasks = [],
- forcedtorun_tasks = [],
- draw_vertically = True,
- ignore_upstream_of_target = False,
- skip_uptodate_tasks = False,
- gnu_make_maximal_rebuild_mode = True,
- test_all_task_for_update = True,
- no_key_legend = False,
- minimal_key_legend = True,
- user_colour_scheme = None,
- pipeline_name = "Pipeline:",
- size = (11,8),
- dpi = 120,
- runtime_data = None,
- checksum_level = None,
- history_file = None):
- # Remember to add further extra parameters here to "extra_pipeline_printout_graph_options" inside cmdline.py
- # This will forward extra parameters from the command line to pipeline_printout_graph
+# _____________________________________________________________________________
+def pipeline_printout_graph(stream,
+ output_format=None,
+ target_tasks=[],
+ forcedtorun_tasks=[],
+ draw_vertically=True,
+ ignore_upstream_of_target=False,
+ skip_uptodate_tasks=False,
+ gnu_make_maximal_rebuild_mode=True,
+ test_all_task_for_update=True,
+ no_key_legend=False,
+ minimal_key_legend=True,
+ user_colour_scheme=None,
+ pipeline_name="Pipeline:",
+ size=(11, 8),
+ dpi = 120,
+ runtime_data = None,
+ checksum_level = None,
+ history_file = None,
+ pipeline = None):
+ # Remember to add further extra parameters here to
+ # "extra_pipeline_printout_graph_options" inside cmdline.py
+ # This will forward extra parameters from the
+ # command line to pipeline_printout_graph
"""
print out pipeline dependencies in various formats
:param stream: where to print to
:type stream: file-like object with ``write()`` function
- :param output_format: ["dot", "jpg", "svg", "ps", "png"]. All but the first depends on the `dot <http://www.graphviz.org>`_ program.
- :param target_tasks: targets task functions which will be run if they are out-of-date.
- :param forcedtorun_tasks: task functions which will be run whether or not they are out-of-date.
+ :param output_format: ["dot", "jpg", "svg", "ps", "png"]. All but the
+ first depends on the
+ `dot <http://www.graphviz.org>`_ program.
+ :param target_tasks: targets task functions which will be run if they are
+ out-of-date.
+ :param forcedtorun_tasks: task functions which will be run whether or not
+ they are out-of-date.
:param draw_vertically: Top to bottom instead of left to right.
:param ignore_upstream_of_target: Don't draw upstream tasks of targets.
:param skip_uptodate_tasks: Don't draw up-to-date tasks if possible.
- :param gnu_make_maximal_rebuild_mode: Defaults to re-running *all* out-of-date tasks. Runs minimal
- set to build targets if set to ``True``. Use with caution.
- :param test_all_task_for_update: Ask all task functions if they are up-to-date.
+ :param gnu_make_maximal_rebuild_mode: Defaults to re-running *all*
+ out-of-date tasks. Runs minimal
+ set to build targets if set to
+ ``True``. Use with caution.
+ :param test_all_task_for_update: Ask all task functions if they are
+ up-to-date.
:param no_key_legend: Don't draw key/legend for graph.
- :param minimal_key_legend: Only add entries to the legend for task types which appear
- :param user_colour_scheme: Dictionary specifying colour scheme for flowchart
+ :param minimal_key_legend: Only legend entries for used task types
+ :param user_colour_scheme: Dictionary specifying flowchart colour scheme
:param pipeline_name: Pipeline Title
:param size: tuple of x and y dimensions
:param dpi: print resolution
- :param runtime_data: Experimental feature for passing data to tasks at run time
- :param history_file: The database file which stores checksums and file timestamps for input/output files.
- :param checksum_level: Several options for checking up-to-dateness are available: Default is level 1.
- level 0 : Use only file timestamps
- level 1 : above, plus timestamp of successful job completion
- level 2 : above, plus a checksum of the pipeline function body
- level 3 : above, plus a checksum of the pipeline function default arguments and the additional arguments passed in by task decorators
+ :param runtime_data: Experimental feature: pass data to tasks at run time
+ :param history_file: Database file storing checksums and file timestamps
+ for input/output files.
+ :param checksum_level: Several options for checking up-to-dateness are
+ available: Default is level 1.
+ level 0 : Use only file timestamps
+ level 1 : above, plus timestamp of successful job completion
+ level 2 : above, plus a checksum of the pipeline function body
+ level 3 : above, plus a checksum of the pipeline function default arguments and the additional arguments passed in by task decorators
"""
# EXTRA pipeline_run DEBUGGING
@@ -2895,92 +4330,127 @@ def pipeline_printout_graph (stream,
if checksum_level is None:
checksum_level = get_default_checksum_level()
- link_task_names_to_functions ()
- update_checksum_level_on_tasks (checksum_level)
+ #
+ # pipeline must be a Pipeline or a string naming a pipeline
+ #
+ pipeline = lookup_pipeline(pipeline)
+ #
+ # Make sure all tasks in dependency list are linked to real functions
+ #
+ processed_tasks = set()
+ completed_pipeline_names = pipeline._complete_task_setup(processed_tasks)
+
+ update_checksum_level_on_tasks(checksum_level)
#
# @active_if decorated tasks can change their active state every time
# pipeline_run / pipeline_printout / pipeline_printout_graph is called
#
- update_active_states_for_all_tasks ()
+ update_active_states_for_all_tasks()
#
# run time data
#
- if runtime_data == None:
+ if runtime_data is None:
runtime_data = {}
if not isinstance(runtime_data, dict):
- raise Exception("pipeline_run parameter runtime_data should be a dictionary of "
- "values passes to jobs at run time.")
+ raise Exception("pipeline_run parameter runtime_data should be a "
+ "dictionary of values passes to jobs at run time.")
#
# If we aren't using checksums, and history file hasn't been specified,
- # we might be a bit surprised to find Ruffus writing to a sqlite db anyway.
- # Let us just use a in memory db which will be thrown away
- # Of course, if history_file is specified, we presume you know what you are doing
+ # we might be a bit surprised to find Ruffus writing to a
+ # sqlite db anyway.
+ # Let us just dump to a placeholder memory db that can then be discarded
+ # Of course, if history_file is specified, we presume you know what
+ # you are doing
#
- if checksum_level == CHECKSUM_FILE_TIMESTAMPS and history_file == None:
+ if checksum_level == CHECKSUM_FILE_TIMESTAMPS and history_file is None:
history_file = ':memory:'
#
# load previous job history if it exists, otherwise create an empty history
#
- job_history = open_job_history (history_file)
+ job_history = open_job_history(history_file)
#
# target jobs
#
- if target_tasks == None:
+ if target_tasks is None:
target_tasks = []
- if forcedtorun_tasks == None:
+ if forcedtorun_tasks is None:
forcedtorun_tasks = []
- target_tasks = task_names_to_tasks ("Target", target_tasks)
- forcedtorun_tasks = task_names_to_tasks ("Forced to run", forcedtorun_tasks)
+ target_tasks = lookup_tasks_from_user_specified_names("Target", target_tasks, pipeline.name)
+ if not target_tasks:
+ target_tasks = list(pipeline.tasks)
+ forcedtorun_tasks = lookup_tasks_from_user_specified_names("Forced to run", forcedtorun_tasks,
+ pipeline.name)
-
- (topological_sorted, ignore_param1, ignore_param2,
- ignore_param3) = topologically_sorted_nodes(target_tasks, forcedtorun_tasks,
- gnu_make_maximal_rebuild_mode,
- extra_data_for_signal = [t_verbose_logger(0, 0, None, runtime_data), job_history])
+ #
+ # forcedtorun_tasks and target_tasks may include more pipelines
+ # which have to be setup
+ #
+ incomplete_pipeline_names = set()
+ for task in forcedtorun_tasks + target_tasks:
+ if task.pipeline.name not in completed_pipeline_names:
+ incomplete_pipeline_names.add(task.pipeline.name)
+
+ for pipeline_name in incomplete_pipeline_names:
+ if pipeline_name in completed_pipeline_names:
+ continue
+ completed_pipeline_names = completed_pipeline_names.union(
+ pipeline.pipelines[pipeline_name]._complete_task_setup(processed_tasks))
+
+ (topological_sorted, ignore_param1, ignore_param2, ignore_param3) = \
+ topologically_sorted_nodes(target_tasks, forcedtorun_tasks,
+ gnu_make_maximal_rebuild_mode,
+ extra_data_for_signal=[
+ t_verbose_logger(0, 0, None, runtime_data), job_history],
+ signal_callback=is_node_up_to_date)
if not len(target_tasks):
target_tasks = topological_sorted[-1:]
-
-
# open file if (unicode?) string
+ close_stream = False
if isinstance(stream, path_str_type):
- stream = open(stream, "w")
+ stream = open(stream, "wb")
+ close_stream = True
# derive format automatically from name
- if output_format == None:
+ if output_format is None:
output_format = os.path.splitext(stream.name)[1].lstrip(".")
-
-
- graph_printout ( stream,
- output_format,
- target_tasks,
- forcedtorun_tasks,
- draw_vertically,
- ignore_upstream_of_target,
- skip_uptodate_tasks,
- gnu_make_maximal_rebuild_mode,
- test_all_task_for_update,
- no_key_legend,
- minimal_key_legend,
- user_colour_scheme,
- pipeline_name,
- size,
- dpi,
- extra_data_for_signal = [t_verbose_logger(0, 0, None, runtime_data), job_history])
-
-
-#_________________________________________________________________________________________
+ try:
+ graph_printout(stream,
+ output_format,
+ target_tasks,
+ forcedtorun_tasks,
+ draw_vertically,
+ ignore_upstream_of_target,
+ skip_uptodate_tasks,
+ gnu_make_maximal_rebuild_mode,
+ test_all_task_for_update,
+ no_key_legend,
+ minimal_key_legend,
+ user_colour_scheme,
+ pipeline_name,
+ size,
+ dpi,
+ extra_data_for_signal=[t_verbose_logger(0, 0, None, runtime_data), job_history],
+ signal_callback=is_node_up_to_date)
+ finally:
+ # if this is a stream we opened, we have to close it ourselves
+ if close_stream:
+ stream.close()
+
+
+# _____________________________________________________________________________
# get_completed_task_strings
-#_________________________________________________________________________________________
-def get_completed_task_strings (incomplete_tasks, all_tasks, forcedtorun_tasks, verbose, verbose_abbreviated_path, indent, runtime_data, job_history):
+# _____________________________________________________________________________
+def get_completed_task_strings(incomplete_tasks, all_tasks, forcedtorun_tasks, verbose,
+ verbose_abbreviated_path, indent, runtime_data, job_history):
"""
Printout list of completed tasks
"""
@@ -2998,7 +4468,9 @@ def get_completed_task_strings (incomplete_tasks, all_tasks, forcedtorun_tasks,
if t in set_of_incomplete_tasks:
continue
# LOGGER
- completed_task_strings.extend(t.printout(runtime_data, t in forcedtorun_tasks, job_history, False, verbose, verbose_abbreviated_path, indent))
+ completed_task_strings.extend(t._printout(runtime_data,
+ t in forcedtorun_tasks, job_history, False,
+ verbose, verbose_abbreviated_path, indent))
completed_task_strings.append("_" * 40)
completed_task_strings.append("")
@@ -3006,31 +4478,37 @@ def get_completed_task_strings (incomplete_tasks, all_tasks, forcedtorun_tasks,
return completed_task_strings
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
# pipeline_printout
-#_________________________________________________________________________________________
-def pipeline_printout( output_stream = None,
- target_tasks = [],
- forcedtorun_tasks = [],
- # verbose defaults to 4 if None
- verbose = None,
- indent = 4,
- gnu_make_maximal_rebuild_mode = True,
- wrap_width = 100,
- runtime_data = None,
- checksum_level = None,
- history_file = None,
- verbose_abbreviated_path = None):
- # Remember to add further extra parameters here to "extra_pipeline_printout_options" inside cmdline.py
- # This will forward extra parameters from the command line to pipeline_printout
+# _____________________________________________________________________________
+
+
+def pipeline_printout(output_stream=None,
+ target_tasks=[],
+ forcedtorun_tasks=[],
+ # verbose defaults to 4 if None
+ verbose=None,
+ indent=4,
+ gnu_make_maximal_rebuild_mode=True,
+ wrap_width=100,
+ runtime_data=None,
+ checksum_level=None,
+ history_file=None,
+ verbose_abbreviated_path=None,
+ pipeline=None):
+ # Remember to add further extra parameters here to
+ # "extra_pipeline_printout_options" inside cmdline.py
+ # This will forward extra parameters from the command
+ # line to pipeline_printout
"""
Printouts the parts of the pipeline which will be run
- Because the parameters of some jobs depend on the results of previous tasks, this function
- produces only the current snap-shot of task jobs. In particular, tasks which generate
- variable number of inputs into following tasks will not produce the full range of jobs.
+ Because the parameters of some jobs depend on the results of previous
+ tasks, this function produces only the current snap-shot of task jobs.
+ In particular, tasks which generate variable number of inputs into
+ following tasks will not produce the full range of jobs.
::
verbose = 0 : Nothing
@@ -3043,8 +4521,10 @@ def pipeline_printout( output_stream = None,
:param output_stream: where to print to
:type output_stream: file-like object with ``write()`` function
- :param target_tasks: targets task functions which will be run if they are out-of-date
- :param forcedtorun_tasks: task functions which will be run whether or not they are out-of-date
+ :param target_tasks: targets task functions which will be run if they are
+ out-of-date
+ :param forcedtorun_tasks: task functions which will be run whether or not
+ they are out-of-date
:param verbose: level 0 : nothing
level 1 : Out-of-date Task names
level 2 : All Tasks (including any task function docstrings)
@@ -3054,20 +4534,23 @@ def pipeline_printout( output_stream = None,
level 6 : All jobs in All Tasks whether out of date or not
level 10: logs messages useful only for debugging ruffus pipeline code
:param indent: How much indentation for pretty format.
- :param gnu_make_maximal_rebuild_mode: Defaults to re-running *all* out-of-date tasks. Runs minimal
- set to build targets if set to ``True``. Use with caution.
+ :param gnu_make_maximal_rebuild_mode: Defaults to re-running *all*
+ out-of-date tasks. Runs minimal
+ set to build targets if set to
+ ``True``. Use with caution.
:param wrap_width: The maximum length of each line
- :param runtime_data: Experimental feature for passing data to tasks at run time
- :param checksum_level: Several options for checking up-to-dateness are available: Default is level 1.
- level 0 : Use only file timestamps
- level 1 : As above, plus timestamp of successful job completion
- level 2 : As above, plus a checksum of the pipeline function body
- level 3 : As above, plus a checksum of the pipeline function default arguments and the additional arguments passed in by task decorators
- :param history_file: The database file which stores checksums and file timestamps for input/output files.
+ :param runtime_data: Experimental feature: pass data to tasks at run time
+ :param checksum_level: Several options for checking up-to-dateness are
+ available: Default is level 1.
+ level 0 : Use only file timestamps
+ level 1 : above, plus timestamp of successful job completion
+ level 2 : above, plus a checksum of the pipeline function body
+ level 3 : above, plus a checksum of the pipeline function default arguments and the additional arguments passed in by task decorators
+ :param history_file: Database file storing checksums and file timestamps for input/output files.
:param verbose_abbreviated_path: whether input and output paths are abbreviated.
- level 0: The full (expanded, abspath) input or output path
- level > 1: The number of subdirectories to include. Abbreviated paths are prefixed with ``[,,,]/``
- level < 0: level < 0: Input / Output parameters are truncated to ``MMM`` letters where ``verbose_abbreviated_path ==-MMM``. Subdirectories are first removed to see if this allows the paths to fit in the specified limit. Otherwise abbreviated paths are prefixed by ``<???>``
+ level 0: The full (expanded, abspath) input or output path
+ level > 1: The number of subdirectories to include. Abbreviated paths are prefixed with ``[,,,]/``
+ level < 0: Input / Output parameters are truncated to ``MMM`` letters where ``verbose_abbreviated_path ==-MMM``. Subdirectories are first removed to see if this allows the paths to fit in the specified limit. Otherwise abbreviated paths are prefixed by ``<???>``
"""
# do nothing!
if verbose == 0:
@@ -3076,70 +4559,101 @@ def pipeline_printout( output_stream = None,
#
# default values
#
- if verbose_abbreviated_path == None:
+ if verbose_abbreviated_path is None:
verbose_abbreviated_path = 2
- if verbose == None:
+ if verbose is None:
verbose = 4
# EXTRA pipeline_run DEBUGGING
global EXTRA_PIPELINERUN_DEBUGGING
EXTRA_PIPELINERUN_DEBUGGING = False
- if output_stream == None:
- import sys
+ if output_stream is None:
output_stream = sys.stdout
-
if not hasattr(output_stream, "write"):
- raise Exception("The first parameter to pipeline_printout needs to be an output file, e.g. sys.stdout and not %s" % str(output_stream))
+ raise Exception("The first parameter to pipeline_printout needs to be "
+ "an output file, e.g. sys.stdout and not %s"
+ % str(output_stream))
- if runtime_data == None:
+ if runtime_data is None:
runtime_data = {}
if not isinstance(runtime_data, dict):
- raise Exception("pipeline_run parameter runtime_data should be a dictionary of "
- "values passes to jobs at run time.")
+ raise Exception("pipeline_run parameter runtime_data should be a "
+ "dictionary of values passes to jobs at run time.")
if checksum_level is None:
checksum_level = get_default_checksum_level()
- link_task_names_to_functions ()
+ #
+ # pipeline must be a Pipeline or a string naming a pipeline
+ #
+ pipeline = lookup_pipeline(pipeline)
+ #
+ # Make sure all tasks in dependency list are linked to real functions
+ #
+ processed_tasks = set()
+ completed_pipeline_names = pipeline._complete_task_setup(processed_tasks)
+
update_checksum_level_on_tasks(checksum_level)
#
# @active_if decorated tasks can change their active state every time
# pipeline_run / pipeline_printout / pipeline_printout_graph is called
#
- update_active_states_for_all_tasks ()
+ update_active_states_for_all_tasks()
#
# target jobs
#
- target_tasks = task_names_to_tasks ("Target", target_tasks)
- forcedtorun_tasks = task_names_to_tasks ("Forced to run", forcedtorun_tasks)
+ target_tasks = lookup_tasks_from_user_specified_names("Target", target_tasks, pipeline.name)
+ if not target_tasks:
+ target_tasks = list(pipeline.tasks)
+ forcedtorun_tasks = lookup_tasks_from_user_specified_names("Forced to run", forcedtorun_tasks,
+ pipeline.name)
- logging_strm = t_verbose_logger(verbose, verbose_abbreviated_path, t_stream_logger(output_stream), runtime_data)
+ #
+ # forcedtorun_tasks and target_tasks may include more pipelines
+ # which have to be setup
+ #
+ incomplete_pipeline_names = set()
+ for task in forcedtorun_tasks + target_tasks:
+ if task.pipeline.name not in completed_pipeline_names:
+ incomplete_pipeline_names.add(task.pipeline.name)
+
+ for pipeline_name in incomplete_pipeline_names:
+ if pipeline_name in completed_pipeline_names:
+ continue
+ completed_pipeline_names = completed_pipeline_names.union(
+ pipeline.pipelines[pipeline_name]._complete_task_setup(processed_tasks))
+
+ logging_strm = t_verbose_logger(verbose, verbose_abbreviated_path,
+ t_stream_logger(output_stream), runtime_data)
#
# If we aren't using checksums, and history file hasn't been specified,
- # we might be a bit surprised to find Ruffus writing to a sqlite db anyway.
- # Let us just use a in memory db which will be thrown away
- # Of course, if history_file is specified, we presume you know what you are doing
- #
- if checksum_level == CHECKSUM_FILE_TIMESTAMPS and history_file == None:
+ # we might be a bit surprised to find Ruffus writing to a
+ # sqlite db anyway.
+ # Let us just dump to a placeholder memory db that can then be discarded
+ # Of course, if history_file is specified, we presume you know what
+ # you are doing
+ if checksum_level == CHECKSUM_FILE_TIMESTAMPS and history_file is None:
history_file = ':memory:'
#
# load previous job history if it exists, otherwise create an empty history
#
- job_history = open_job_history (history_file)
+ job_history = open_job_history(history_file)
(incomplete_tasks,
- self_terminated_nodes,
- dag_violating_edges,
- dag_violating_nodes) = topologically_sorted_nodes(target_tasks, forcedtorun_tasks,
- gnu_make_maximal_rebuild_mode,
- extra_data_for_signal = [t_verbose_logger(0, 0, None, runtime_data), job_history])
-
+ self_terminated_nodes,
+ dag_violating_edges,
+ dag_violating_nodes) = \
+ topologically_sorted_nodes(target_tasks, forcedtorun_tasks,
+ gnu_make_maximal_rebuild_mode,
+ extra_data_for_signal=[
+ t_verbose_logger(0, 0, None, runtime_data), job_history],
+ signal_callback=is_node_up_to_date)
#
# raise error if DAG violating nodes
@@ -3147,9 +4661,8 @@ def pipeline_printout( output_stream = None,
if len(dag_violating_nodes):
dag_violating_tasks = ", ".join(t._name for t in dag_violating_nodes)
- e = error_circular_dependencies("Circular dependencies found in the "
- "pipeline involving one or more of (%s)" %
- (dag_violating_tasks))
+ e = error_circular_dependencies("Circular dependencies found in the pipeline involving "
+ "one or more of (%s)" % (dag_violating_tasks,))
raise e
wrap_indent = " " * (indent + 11)
@@ -3159,67 +4672,190 @@ def pipeline_printout( output_stream = None,
#
# LOGGER level 6 : All jobs in All Tasks whether out of date or not
if verbose == 2 or verbose >= 5:
- (all_tasks, ignore_param1, ignore_param2,
- ignore_param3) = topologically_sorted_nodes(target_tasks, True,
- gnu_make_maximal_rebuild_mode,
- extra_data_for_signal = [t_verbose_logger(0, 0, None, runtime_data), job_history])
- for m in get_completed_task_strings (incomplete_tasks, all_tasks, forcedtorun_tasks, verbose, verbose_abbreviated_path, indent, runtime_data, job_history):
- output_stream.write(textwrap.fill(m, subsequent_indent = wrap_indent, width = wrap_width) + "\n")
+ (all_tasks, ignore_param1, ignore_param2, ignore_param3) = \
+ topologically_sorted_nodes(target_tasks, True, gnu_make_maximal_rebuild_mode,
+ extra_data_for_signal=[
+ t_verbose_logger(0, 0, None, runtime_data),
+ job_history],
+ signal_callback=is_node_up_to_date)
+ for m in get_completed_task_strings(incomplete_tasks, all_tasks, forcedtorun_tasks,
+ verbose, verbose_abbreviated_path, indent,
+ runtime_data, job_history):
+ output_stream.write(textwrap.fill(m, subsequent_indent=wrap_indent,
+ width=wrap_width) + "\n")
output_stream.write("\n" + "_" * 40 + "\nTasks which will be run:\n\n")
for t in incomplete_tasks:
# LOGGER
- messages = t.printout(runtime_data, t in forcedtorun_tasks, job_history, True, verbose, verbose_abbreviated_path, indent)
+ messages = t._printout(runtime_data, t in forcedtorun_tasks,
+ job_history, True, verbose,
+ verbose_abbreviated_path, indent)
for m in messages:
- output_stream.write(textwrap.fill(m, subsequent_indent = wrap_indent, width = wrap_width) + "\n")
+ output_stream.write(textwrap.fill(m, subsequent_indent=wrap_indent,
+ width=wrap_width) + "\n")
if verbose:
# LOGGER
output_stream.write("_" * 40 + "\n")
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
# get_semaphore
-#_________________________________________________________________________________________
-def get_semaphore (t, job_limit_semaphores, syncmanager):
+# _____________________________________________________________________________
+def get_semaphore(t, _job_limit_semaphores, syncmanager):
"""
return semaphore to limit the number of concurrent jobs
"""
#
# Is this task limited in the number of jobs?
#
- if t.semaphore_name not in t.job_limit_semaphores:
+ if t.semaphore_name not in t._job_limit_semaphores:
return None
-
#
# create semaphore if not yet created
#
- if t.semaphore_name not in job_limit_semaphores:
- maximum_jobs_num = t.job_limit_semaphores[t.semaphore_name]
- job_limit_semaphores[t.semaphore_name] = syncmanager.BoundedSemaphore(maximum_jobs_num)
- return job_limit_semaphores[t.semaphore_name]
+ if t.semaphore_name not in _job_limit_semaphores:
+ maximum_jobs_num = t._job_limit_semaphores[t.semaphore_name]
+ _job_limit_semaphores[t.semaphore_name] = syncmanager.BoundedSemaphore(maximum_jobs_num)
+ return _job_limit_semaphores[t.semaphore_name]
+
+
+# _____________________________________________________________________________
+
+# job_needs_to_run
+
+# Helper function for make_job_parameter_generator
+# _____________________________________________________________________________
+def job_needs_to_run(task, params, force_rerun, logger, verbose, job_name,
+ job_history, verbose_abbreviated_path):
+ """
+ Check if job parameters out of date / needs to rerun
+ """
+
+ #
+ # Out of date because forced to run
+ #
+ if force_rerun:
+ # LOGGER: Out-of-date Jobs in Out-of-date Tasks
+ log_at_level(logger, 3, verbose, " force task %s to rerun "
+ % job_name)
+ return True
+
+ if not task.needs_update_func:
+ # LOGGER: Out-of-date Jobs in Out-of-date Tasks
+ log_at_level(logger, 3, verbose, " %s no function to check "
+ "if up-to-date " % job_name)
+ return True
+
+ # extra clunky hack to also pass task info--
+ # makes sure that there haven't been code or
+ # arg changes
+ if task.needs_update_func == needs_update_check_modify_time:
+ needs_update, msg = task.needs_update_func(
+ *params, task=task, job_history=job_history,
+ verbose_abbreviated_path=verbose_abbreviated_path)
+ else:
+ needs_update, msg = task.needs_update_func(*params)
+
+ if not needs_update:
+ # LOGGER: All Jobs in Out-of-date Tasks
+ log_at_level(logger, 5, verbose,
+ " %s unnecessary: already up to date " % job_name)
+ return False
+
+ # LOGGER: Out-of-date Jobs in Out-of-date
+ # Tasks: Why out of date
+ if not log_at_level(logger, 4, verbose, " %s %s " % (job_name, msg)):
+ # LOGGER: Out-of-date Jobs in
+ # Out-of-date Tasks: No explanation
+ log_at_level(logger, 3, verbose, " %s" % (job_name))
+
+ #
+ # Clunky hack to make sure input files exists right
+ # before job is called for better error messages
+ #
+ if task.needs_update_func == needs_update_check_modify_time:
+ check_input_files_exist(*params)
+
+ return True
+
+
+# _____________________________________________________________________________
+#
+# remove_completed_tasks
+#
+# Helper function for make_job_parameter_generator
+# _____________________________________________________________________________
+def remove_completed_tasks(task_with_completed_job_q, incomplete_tasks,
+ count_remaining_jobs, logger, verbose):
+ """
+ Remove completed tasks in same thread as job parameters generation to
+ prevent race conditions
+ Task completion is usually signalled from pipeline_run
+ """
+ while True:
+ try:
+ (job_completed_task,
+ job_completed_task_name,
+ job_completed_node_index,
+ job_completed_name) = task_with_completed_job_q.get_nowait()
+
+ if job_completed_task not in incomplete_tasks:
+ raise Exception("Last job %s for %s. Missing from "
+ "incomplete tasks in make_job_parameter_generator"
+ % (job_completed_name, job_completed_task_name))
+ count_remaining_jobs[job_completed_task] -= 1
+ #
+ # Negative job count : something has gone very wrong
+ #
+ if count_remaining_jobs[job_completed_task] < 0:
+ raise Exception("job %s for %s causes job count < 0."
+ % (job_completed_name,
+ job_completed_task_name))
+
+ #
+ # This Task completed
+ #
+ if count_remaining_jobs[job_completed_task] == 0:
+ log_at_level(logger, 10, verbose, " Last job for %r. "
+ "Retired from incomplete tasks in pipeline_run "
+ % job_completed_task._get_display_name())
+ incomplete_tasks.remove(job_completed_task)
+ job_completed_task._completed()
+ log_at_level(logger, 1, verbose, "Completed Task = %r "
+ % job_completed_task._get_display_name())
+
+ except queue.Empty:
+ break
+
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
#
-# Parameter generator for all jobs / tasks
+# Parameter generator factory for all jobs / tasks
#
-#________________________________________________________________________________________
-def make_job_parameter_generator (incomplete_tasks, task_parents, logger, forcedtorun_tasks,
- task_with_completed_job_q, runtime_data, verbose,
- verbose_abbreviated_path,
- syncmanager,
- death_event,
- touch_files_only, job_history):
+# _____________________________________________________________________________
+def make_job_parameter_generator(incomplete_tasks, task_parents, logger,
+ forcedtorun_tasks, task_with_completed_job_q,
+ runtime_data, verbose,
+ verbose_abbreviated_path,
+ syncmanager,
+ death_event,
+ touch_files_only, job_history):
inprogress_tasks = set()
- job_limit_semaphores = dict()
+ _job_limit_semaphores = dict()
+ # _________________________________________________________________________
+ #
+ # Parameter generator returned by factory
+ #
+ # _________________________________________________________________________
def parameter_generator():
count_remaining_jobs = defaultdict(int)
- log_at_level (logger, 10, verbose, " job_parameter_generator BEGIN")
+ log_at_level(logger, 10, verbose, " job_parameter_generator BEGIN")
while len(incomplete_tasks):
cnt_jobs_created_for_all_tasks = 0
cnt_tasks_processed = 0
@@ -3228,46 +4864,23 @@ def make_job_parameter_generator (incomplete_tasks, task_parents, logger, forced
# get rid of all completed tasks first
# Completion is signalled from pipeline_run
#
- while True:
- try:
- item = task_with_completed_job_q.get_nowait()
- job_completed_task, job_completed_task_name, job_completed_name = item
-
-
- if not job_completed_task in incomplete_tasks:
- raise Exception("Last job %s for %s. Missing from incomplete tasks in make_job_parameter_generator" % (job_completed_name, job_completed_task_name))
- count_remaining_jobs[job_completed_task] = count_remaining_jobs[job_completed_task] - 1
- #
- # This is bad: something has gone very wrong
- #
- if count_remaining_jobs[t] < 0:
- raise Exception("job %s for %s causes job count < 0." % (job_completed_name, job_completed_task_name))
-
- #
- # This Task completed
- #
- if count_remaining_jobs[job_completed_task] == 0:
- log_at_level (logger, 10, verbose, " Last job for %s. Retired from incomplete tasks in pipeline_run " % job_completed_task._name)
- incomplete_tasks.remove(job_completed_task)
- job_completed_task.completed ()
- # LOGGER: Out-of-date Tasks
- log_at_level (logger, 1, verbose, "Completed Task = " + job_completed_task.get_task_name())
-
- except queue.Empty:
- break
+ remove_completed_tasks(task_with_completed_job_q, incomplete_tasks,
+ count_remaining_jobs, logger, verbose)
for t in list(incomplete_tasks):
#
- # wrap in execption handler so that we know which task exception
- # came from
+ # wrap in execption handler so that we know
+ # which task the original exception came from
#
try:
- log_at_level (logger, 10, verbose, " job_parameter_generator consider task = %s" % t._name)
+ log_at_level(logger, 10, verbose, " job_parameter_generator consider "
+ "task = %r" % t._get_display_name())
# ignore tasks in progress
if t in inprogress_tasks:
continue
- log_at_level (logger, 10, verbose, " job_parameter_generator task %s not in progress" % t._name)
+ log_at_level(logger, 10, verbose, " job_parameter_generator task %r not in "
+ "progress" % t._get_display_name())
# ignore tasks with incomplete dependencies
incomplete_parent = False
@@ -3278,266 +4891,251 @@ def make_job_parameter_generator (incomplete_tasks, task_parents, logger, forced
if incomplete_parent:
continue
- log_at_level (logger, 10, verbose, " job_parameter_generator start task %s (parents completed)" % t._name)
+ log_at_level(logger, 10, verbose, " job_parameter_generator start task %r "
+ "(parents completed)" % t._get_display_name())
force_rerun = t in forcedtorun_tasks
- #
- # Only log active task
- #
- if t.is_active:
- # LOGGER: Out-of-date Tasks
- log_at_level (logger, 1, verbose, "Task enters queue = " + t.get_task_name() + (": Forced to rerun" if force_rerun else ""))
- # LOGGER: logs All Tasks (including any task function docstrings)
- # indent string
- if len(t._description):
- log_at_level (logger, 2, verbose, " " + t._description)
inprogress_tasks.add(t)
cnt_tasks_processed += 1
-
#
- # Use output parameters actually generated by running task
+ # Log active task
#
- t.output_filenames = []
-
-
-
+ if t.is_active:
+ forced_msg = ": Forced to rerun" if force_rerun else ""
+ log_at_level(logger, 1, verbose, "Task enters queue = %r %s"
+ % (t._get_display_name(), forced_msg))
+ if len(t.func_description):
+ log_at_level(logger, 2, verbose, " " + t.func_description)
#
- # If no parameters: just call task function (empty list)
+ # Inactive skip loop
#
- #if (t.active_if_checks != None):
- # t.is_active = all(arg() if isinstance(arg, collections.Callable) else arg
- # for arg in t.active_if_checks)
- if not t.is_active:
- parameters = []
-
+ else:
+ incomplete_tasks.remove(t)
+ # N.B. inactive tasks are not _completed()
+ # t._completed()
+ t.output_filenames = None
+ log_at_level(logger, 2, verbose, "Inactive Task = %r"
+ % t._get_display_name())
+ continue
+ #
+ # Use output parameters generated by running task
+ #
+ t.output_filenames = []
#
# If no parameters: just call task function (empty list)
#
- elif t.param_generator_func == None:
- parameters = ([[], []],)
+ if t.param_generator_func is None:
+ task_parameters = ([[], []],)
else:
- parameters = t.param_generator_func(runtime_data)
+ task_parameters = t.param_generator_func(runtime_data)
#
- # iterate through parameters
+ # iterate through jobs
#
cnt_jobs_created = 0
- for param, descriptive_param in parameters:
+ for params, unglobbed_params in task_parameters:
+
#
# save output even if uptodate
#
- if len(param) >= 2:
- t.output_filenames.append(param[1])
+ if len(params) >= 2:
+ # To do: In the case of split subdivide, we should be doing this after
+ # The job finishes
+ t.output_filenames.append(params[1])
- job_name = t.get_job_name(descriptive_param, verbose_abbreviated_path, runtime_data)
+ job_name = t._get_job_name(unglobbed_params,
+ verbose_abbreviated_path,
+ runtime_data)
- #
- # don't run if up to date unless force to run
- #
- if force_rerun:
- # LOGGER: Out-of-date Jobs in Out-of-date Tasks
- log_at_level (logger, 3, verbose, " force task %s to rerun " % job_name)
- else:
- if not t.needs_update_func:
- # LOGGER: Out-of-date Jobs in Out-of-date Tasks
- log_at_level (logger, 3, verbose, " %s no function to check if up-to-date " % job_name)
- else:
- # extra clunky hack to also pass task info--
- # makes sure that there haven't been code or arg changes
- if t.needs_update_func == needs_update_check_modify_time:
- needs_update, msg = t.needs_update_func (*param, task=t, job_history = job_history, verbose_abbreviated_path = verbose_abbreviated_path)
- else:
- needs_update, msg = t.needs_update_func (*param)
-
- if not needs_update:
- # LOGGER: All Jobs in Out-of-date Tasks
- log_at_level (logger, 5, verbose, " %s unnecessary: already up to date " % job_name)
- continue
- else:
- # LOGGER: Out-of-date Jobs in Out-of-date Tasks: Why out of date
- if not log_at_level (logger, 4, verbose, " %s %s " % (job_name, msg)):
-
- # LOGGER: Out-of-date Jobs in Out-of-date Tasks: No explanation
- log_at_level (logger, 3, verbose, " %s" % (job_name))
-
- #
- # Clunky hack to make sure input files exists right before
- # job is called for better error messages
- #
- if t.needs_update_func == needs_update_check_modify_time:
- check_input_files_exist (*param)
+ if not job_needs_to_run(t, params, force_rerun, logger, verbose, job_name,
+ job_history, verbose_abbreviated_path):
+ continue
# pause for one second before first job of each tasks
- # @originate tasks do not need to pause, because they depend on nothing!
+ # @originate tasks do not need to pause,
+ # because they depend on nothing!
if cnt_jobs_created == 0 and touch_files_only < 2:
- if "ONE_SECOND_PER_JOB" in runtime_data and runtime_data["ONE_SECOND_PER_JOB"] and t._action_type != _task.action_task_originate:
- log_at_level (logger, 10, verbose, " 1 second PAUSE in job_parameter_generator\n\n\n")
+ if "ONE_SECOND_PER_JOB" in runtime_data and \
+ runtime_data["ONE_SECOND_PER_JOB"] and \
+ t._action_type != Task._action_task_originate:
+ log_at_level(logger, 10, verbose,
+ " 1 second PAUSE in job_parameter_generator\n\n\n")
time.sleep(1.01)
else:
time.sleep(0.1)
-
count_remaining_jobs[t] += 1
cnt_jobs_created += 1
cnt_jobs_created_for_all_tasks += 1
- yield (param,
- t._name,
- job_name,
- t.job_wrapper,
- t.user_defined_work_func,
- get_semaphore (t, job_limit_semaphores, syncmanager),
- death_event,
- touch_files_only)
+
+ yield (params,
+ unglobbed_params,
+ t._name,
+ t._node_index,
+ job_name,
+ t.job_wrapper,
+ t.user_defined_work_func,
+ get_semaphore(t, _job_limit_semaphores, syncmanager),
+ death_event,
+ touch_files_only)
# if no job came from this task, this task is complete
- # we need to retire it here instead of normal completion at end of job tasks
- # precisely because it created no jobs
+ # we need to retire it here instead of normal completion
+ # at end of job tasks precisely
+ # because it created no jobs
if cnt_jobs_created == 0:
incomplete_tasks.remove(t)
- t.completed ()
- if not t.is_active:
- log_at_level (logger, 2, verbose, "Inactive Task = " + t.get_task_name())
- else:
- log_at_level (logger, 2, verbose, "Uptodate Task = " + t.get_task_name())
+ t._completed()
+ log_at_level(logger, 2, verbose,
+ "Uptodate Task = %r" % t._get_display_name())
# LOGGER: logs All Tasks (including any task function docstrings)
- log_at_level (logger, 10, verbose, " No jobs created for %s. Retired in parameter_generator " % t._name)
+ log_at_level(logger, 10, verbose, " No jobs created for %r. Retired "
+ "in parameter_generator " % t._get_display_name())
#
# Add extra warning if no regular expressions match:
# This is a common class of frustrating errors
#
- if (verbose >= 1 and "ruffus_WARNING" in runtime_data and
- t.param_generator_func in runtime_data["ruffus_WARNING"]):
- for msg in runtime_data["ruffus_WARNING"][t.param_generator_func]:
- logger.warning(" 'In Task def %s(...):' %s " % (t.get_task_name(), msg))
-
+ if verbose >= 1 and \
+ "ruffus_WARNING" in runtime_data and \
+ t.param_generator_func in runtime_data["ruffus_WARNING"]:
+ for msg in runtime_data["ruffus_WARNING"][
+ t.param_generator_func]:
+ logger.warning(" 'In Task %r:' %s "
+ % (t._get_display_name(), msg))
#
- # GeneratorExit is thrown when this generator does not complete.
+ # GeneratorExit thrown when generator doesn't complete.
# I.e. there is a break in the pipeline_run loop.
- # This happens where there are exceptions signalled from within a job
+ # This happens where there are exceptions
+ # signalled from within a job
#
- # This is not really an exception, more a way to exit the generator loop
- # asynchrononously so that cleanups can happen (e.g. the "with" statement
- # or finally.)
+ # This is not really an exception, more a way to exit the
+ # generator loop asynchrononously so that cleanups can
+ # happen (e.g. the "with" statement or finally.)
#
- # We could write except Exception: below which will catch everything but
- # KeyboardInterrupt and StopIteration and GeneratorExit in python 2.6
+ # We could write except Exception: below which will catch
+ # everything but KeyboardInterrupt and StopIteration
+ # and GeneratorExit in python 2.6
#
- # However, in python 2.5, GeneratorExit inherits from Exception. So
- # we explicitly catch and rethrow GeneratorExit.
+ # However, in python 2.5, GeneratorExit inherits from
+ # Exception. So we explicitly catch and rethrow
+ # GeneratorExit.
except GeneratorExit:
raise
except:
exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
- exception_stack = traceback.format_exc()
- exception_name = exceptionType.__module__ + '.' + exceptionType.__name__
- exception_value = str(exceptionValue)
+ exception_stack = traceback.format_exc()
+ exception_name = exceptionType.__module__ + '.' + exceptionType.__name__
+ exception_value = str(exceptionValue)
if len(exception_value):
exception_value = "(%s)" % exception_value
errt = RethrownJobError([(t._name,
- "",
- exception_name,
- exception_value,
- exception_stack)])
+ "",
+ exception_name,
+ exception_value,
+ exception_stack)])
errt.specify_task(t, "Exceptions generating parameters")
raise errt
-
-
# extra tests incase final tasks do not result in jobs
- if len(incomplete_tasks) and (not cnt_tasks_processed or cnt_jobs_created_for_all_tasks):
- log_at_level (logger, 10, verbose, " incomplete tasks = " +
- ",".join([t._name for t in incomplete_tasks] ))
+ if len(incomplete_tasks) and \
+ (not cnt_tasks_processed or cnt_jobs_created_for_all_tasks):
+ log_at_level(logger, 10, verbose, " incomplete tasks = " +
+ ",".join([t._name for t in incomplete_tasks]))
yield waiting_for_more_tasks_to_complete()
yield all_tasks_complete()
# This function is done
- log_at_level (logger, 10, verbose, " job_parameter_generator END")
+ log_at_level(logger, 10, verbose, " job_parameter_generator END")
return parameter_generator
-
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
#
# feed_job_params_to_process_pool
#
#
-#________________________________________________________________________________________
-def feed_job_params_to_process_pool_factory (parameter_q, death_event, logger, verbose):
+# _____________________________________________________________________________
+def feed_job_params_to_process_pool_factory(parameter_q, death_event, logger,
+ verbose):
"""
Process pool gets its parameters from this generator
Use factory function to save parameter_queue
"""
- def feed_job_params_to_process_pool ():
- log_at_level (logger, 10, verbose, " Send param to Pooled Process START")
+ def feed_job_params_to_process_pool():
+ log_at_level(logger, 10, verbose, " Send params to Pooled Process START")
while 1:
- log_at_level (logger, 10, verbose, " Get next parameter size = %d" %
- parameter_q.qsize())
+ log_at_level(logger, 10, verbose,
+ " Get next parameter size = %d" % parameter_q.qsize())
if not parameter_q.qsize():
time.sleep(0.1)
- param = parameter_q.get()
- log_at_level (logger, 10, verbose, " Get next parameter done")
+ params = parameter_q.get()
+ log_at_level(logger, 10, verbose, " Get next parameter done")
# all tasks done
- if isinstance(param, all_tasks_complete):
+ if isinstance(params, all_tasks_complete):
break
if death_event.is_set():
death_event.clear()
break
- log_at_level (logger, 10, verbose, " Send param to Pooled Process=>" + str(param[0]))
- yield param
+ log_at_level(logger, 10, verbose,
+ " Send params to Pooled Process=>" + str(params[0]))
+ yield params
- log_at_level (logger, 10, verbose, " Send param to Pooled Process END")
+ log_at_level(logger, 10, verbose, " Send params to Pooled Process END")
# return generator
return feed_job_params_to_process_pool
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
#
# fill_queue_with_job_parameters
#
-#________________________________________________________________________________________
-def fill_queue_with_job_parameters (job_parameters, parameter_q, POOL_SIZE, logger, verbose):
+# _____________________________________________________________________________
+
+
+def fill_queue_with_job_parameters(job_parameters, parameter_q, POOL_SIZE,
+ logger, verbose):
"""
- Ensures queue is filled with number of parameters > jobs / slots (POOL_SIZE)
+ Ensures queue filled with number of parameters > jobs / slots (POOL_SIZE)
"""
- log_at_level (logger, 10, verbose, " fill_queue_with_job_parameters START")
- for param in job_parameters:
+ log_at_level(logger, 10, verbose, " fill_queue_with_job_parameters START")
+ for params in job_parameters:
# stop if no more jobs available
- if isinstance(param, waiting_for_more_tasks_to_complete):
- log_at_level (logger, 10, verbose, " fill_queue_with_job_parameters WAITING for task to complete")
+ if isinstance(params, waiting_for_more_tasks_to_complete):
+ log_at_level(logger, 10, verbose,
+ " fill_queue_with_job_parameters WAITING for task to complete")
break
- if not isinstance(param, all_tasks_complete):
- log_at_level (logger, 10, verbose, " fill_queue_with_job_parameters=>" + str(param[0]))
+ if not isinstance(params, all_tasks_complete):
+ log_at_level(logger, 10, verbose, " fill_queue_with_job_parameters=>" +
+ str(params[0]))
# put into queue
- parameter_q.put(param)
+ parameter_q.put(params)
- # queue size needs to be at least 2 so that the parameter queue never consists of a single
- # waiting_for_task_to_complete entry which will cause
- # a loop and everything to hang!
+ # queue size needs to be at least 2 so that the parameter queue never
+ # consists of a singlewaiting_for_task_to_complete entry which will
+ # cause a loop and everything to hang!
if parameter_q.qsize() > POOL_SIZE + 1:
break
- log_at_level (logger, 10, verbose, " fill_queue_with_job_parameters END")
+ log_at_level(logger, 10, verbose, " fill_queue_with_job_parameters END")
-
-
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
# pipeline_get_task_names
-#_________________________________________________________________________________________
-def pipeline_get_task_names ():
+# _____________________________________________________________________________
+def pipeline_get_task_names(pipeline=None):
"""
Get all task names in a pipeline
Not that does not check if pipeline is wired up properly
@@ -3548,17 +5146,69 @@ def pipeline_get_task_names ():
EXTRA_PIPELINERUN_DEBUGGING = False
#
+ # pipeline must be a Pipeline or a string naming a pipeline
+ #
+ pipeline = lookup_pipeline(pipeline)
+
+ #
# Make sure all tasks in dependency list are linked to real functions
#
- link_task_names_to_functions ()
+ processed_tasks = set()
+ completed_pipeline_names = pipeline._complete_task_setup(processed_tasks)
#
# Return task names for all nodes willy nilly
#
+ return [n._name for n in node._all_nodes]
+
+
+# _____________________________________________________________________________
+
+# get_job_result_output_file_names
- return [n.get_task_name() for n in node._all_nodes]
+# _____________________________________________________________________________
+def get_job_result_output_file_names(job_result):
+ """
+ Excludes input file names being passed through
+ """
+ if len(job_result.unglobbed_params) <= 1: # some jobs have no outputs
+ return
+
+ unglobbed_input_params = job_result.unglobbed_params[0]
+ unglobbed_output_params = job_result.unglobbed_params[1]
+
+ # some have multiple outputs from one job
+ if not isinstance(unglobbed_output_params, list):
+ unglobbed_output_params = [unglobbed_output_params]
+
+ # canonical path of input files, retaining any symbolic links:
+ # symbolic links have their own checksumming
+ input_file_names = set()
+ for i_f_n in get_strings_in_flattened_sequence([unglobbed_input_params]):
+ input_file_names.add(os.path.abspath(i_f_n))
+
+ #
+ # N.B. output parameters are not necessary all strings
+ # and not all files have been successfully created,
+ # even though the task apparently completed properly!
+ # Remember to re-expand globs (from unglobbed paramters)
+ # after the job has run successfully
+ #
+ for possible_glob_str in get_strings_in_flattened_sequence(unglobbed_output_params):
+ for o_f_n in glob.glob(possible_glob_str):
+ #
+ # Exclude output files if they are input files "passed through"
+ #
+ if os.path.abspath(o_f_n) in input_file_names:
+ continue
+ #
+ # use paths relative to working directory
+ #
+ yield os.path.relpath(o_f_n)
+
+ return
#
# How the job queue works:
@@ -3573,89 +5223,132 @@ def pipeline_get_task_names ():
# else:
#
# loops through jobs until no more jobs in non-dependent tasks
-# separate loop in generator so that list of incomplete_tasks does not
-# get updated half way through
+# separate loop in generator so that list of incomplete_tasks
+# does not get updated half way through
# causing race conditions
#
-# parameter_q.put(param)
+# parameter_q.put(params)
# until waiting_for_more_tasks_to_complete
# until queue is full (check *after*)
#
-#_________________________________________________________________________________________
+# _____________________________________________________________________________
# pipeline_run
-#_________________________________________________________________________________________
-def pipeline_run(target_tasks = [],
- forcedtorun_tasks = [],
- multiprocess = 1,
- logger = stderr_logger,
- gnu_make_maximal_rebuild_mode = True,
- #verbose defaults to 1 if None
- verbose = None,
- runtime_data = None,
- one_second_per_job = None,
- touch_files_only = False,
- exceptions_terminate_immediately = False,
- log_exceptions = False,
- checksum_level = None,
- multithread = 0,
- history_file = None,
+# _____________________________________________________________________________
+def pipeline_run(target_tasks=[],
+ forcedtorun_tasks=[],
+ multiprocess=1,
+ logger=stderr_logger,
+ gnu_make_maximal_rebuild_mode=True,
+ # verbose defaults to 1 if None
+ verbose=None,
+ runtime_data=None,
+ one_second_per_job=None,
+ touch_files_only=False,
+ exceptions_terminate_immediately=False,
+ log_exceptions=False,
+ checksum_level=None,
+ multithread=0,
+ history_file=None,
# defaults to 2 if None
- verbose_abbreviated_path = None):
- # Remember to add further extra parameters here to "extra_pipeline_run_options" inside cmdline.py
- # This will forward extra parameters from the command line to pipeline_run
+ verbose_abbreviated_path=None,
+ pipeline=None):
+ # Remember to add further extra parameters here to
+ # "extra_pipeline_run_options" inside cmdline.py
+ # This will forward extra parameters from the command line to
+ # pipeline_run
"""
Run pipelines.
- :param target_tasks: targets task functions which will be run if they are out-of-date
- :param forcedtorun_tasks: task functions which will be run whether or not they are out-of-date
- :param multiprocess: The number of concurrent jobs running on different processes.
- :param multithread: The number of concurrent jobs running as different threads. If > 1, ruffus will use multithreading *instead of* multiprocessing (and ignore the multiprocess parameter). Using multi threading is particularly useful to manage high performance clusters which otherwise are prone to "processor storms" when large number of cores finish jobs at the same time. (Thanks Andreas Heger)
+ :param target_tasks: targets task functions which will be run if they are
+ out-of-date
+ :param forcedtorun_tasks: task functions which will be run whether or not
+ they are out-of-date
+ :param multiprocess: The number of concurrent jobs running on different
+ processes.
+ :param multithread: The number of concurrent jobs running as different
+ threads. If > 1, ruffus will use multithreading
+ *instead of* multiprocessing (and ignore the
+ multiprocess parameter). Using multi threading
+ is particularly useful to manage high performance
+ clusters which otherwise are prone to
+ "processor storms" when large number of cores finish
+ jobs at the same time. (Thanks Andreas Heger)
:param logger: Where progress will be logged. Defaults to stderr output.
- :type logger: `logging <http://docs.python.org/library/logging.html>`_ objects
- :param verbose: level 0 : nothing
- level 1 : Out-of-date Task names
- level 2 : All Tasks (including any task function docstrings)
- level 3 : Out-of-date Jobs in Out-of-date Tasks, no explanation
- level 4 : Out-of-date Jobs in Out-of-date Tasks, with explanations and warnings
- level 5 : All Jobs in Out-of-date Tasks, (include only list of up-to-date tasks)
- level 6 : All jobs in All Tasks whether out of date or not
- level 10: logs messages useful only for debugging ruffus pipeline code
- :param touch_files_only: Create or update input/output files only to simulate running the pipeline. Do not run jobs. If set to CHECKSUM_REGENERATE, will regenerate the checksum history file to reflect the existing i/o files on disk.
- :param exceptions_terminate_immediately: Exceptions cause immediate termination
- rather than waiting for N jobs to finish where N = multiprocess
- :param log_exceptions: Print exceptions to the logger as soon as they occur.
- :param checksum_level: Several options for checking up-to-dateness are available: Default is level 1.
- level 0 : Use only file timestamps
- level 1 : above, plus timestamp of successful job completion
- level 2 : above, plus a checksum of the pipeline function body
- level 3 : above, plus a checksum of the pipeline function default arguments and the additional arguments passed in by task decorators
- :param one_second_per_job: To work around poor file timepstamp resolution for some file systems. Defaults to True if checksum_level is 0 forcing Tasks to take a minimum of 1 second to complete.
- :param runtime_data: Experimental feature for passing data to tasks at run time
- :param gnu_make_maximal_rebuild_mode: Defaults to re-running *all* out-of-date tasks. Runs minimal
- set to build targets if set to ``True``. Use with caution.
- :param history_file: The database file which stores checksums and file timestamps for input/output files.
+ :type logger: `logging <http://docs.python.org/library/logging.html>`_
+ objects
+ :param verbose:
+
+ * level 0 : nothing
+ * level 1 : Out-of-date Task names
+ * level 2 : All Tasks (including any task function docstrings)
+ * level 3 : Out-of-date Jobs in Out-of-date Tasks, no explanation
+ * level 4 : Out-of-date Jobs in Out-of-date Tasks, with explanations and warnings
+ * level 5 : All Jobs in Out-of-date Tasks, (include only list of up-to-date
+ tasks)
+ * level 6 : All jobs in All Tasks whether out of date or not
+ * level 10: logs messages useful only for debugging ruffus pipeline code
+ :param touch_files_only: Create or update input/output files only to
+ simulate running the pipeline. Do not run jobs.
+ If set to CHECKSUM_REGENERATE, will regenerate
+ the checksum history file to reflect the existing
+ i/o files on disk.
+ :param exceptions_terminate_immediately: Exceptions cause immediate
+ termination rather than waiting
+ for N jobs to finish where
+ N = multiprocess
+ :param log_exceptions: Print exceptions to logger as soon as they occur.
+ :param checksum_level: Several options for checking up-to-dateness are
+ available: Default is level 1.
+
+ * level 0 : Use only file timestamps
+ * level 1 : above, plus timestamp of successful job completion
+ * level 2 : above, plus a checksum of the pipeline function body
+ * level 3 : above, plus a checksum of the pipeline
+ function default arguments and the
+ additional arguments passed in by task
+ decorators
+ :param one_second_per_job: To work around poor file timepstamp resolution
+ for some file systems. Defaults to True if
+ checksum_level is 0 forcing Tasks to take a
+ minimum of 1 second to complete.
+ :param runtime_data: Experimental feature: pass data to tasks at run time
+ :param gnu_make_maximal_rebuild_mode: Defaults to re-running *all*
+ out-of-date tasks. Runs minimal
+ set to build targets if set to
+ ``True``. Use with caution.
+ :param history_file: Database file storing checksums and file timestamps
+ for input/output files.
:param verbose_abbreviated_path: whether input and output paths are abbreviated.
- level 0: The full (expanded, abspath) input or output path
- level > 1: The number of subdirectories to include. Abbreviated paths are prefixed with ``[,,,]/``
- level < 0: level < 0: Input / Output parameters are truncated to ``MMM`` letters where ``verbose_abbreviated_path ==-MMM``. Subdirectories are first removed to see if this allows the paths to fit in the specified limit. Otherwise abbreviated paths are prefixed by ``<???>``
+
+ * level 0: The full (expanded, abspath) input or output path
+ * level > 1: The number of subdirectories to include.
+ Abbreviated paths are prefixed with ``[,,,]/``
+ * level < 0: Input / Output parameters are truncated
+ to ``MMM`` letters where ``verbose_abbreviated_path
+ ==-MMM``. Subdirectories are first removed to see
+ if this allows the paths to fit in the specified
+ limit. Otherwise abbreviated paths are prefixed by
+ ``<???>``
"""
+ # DEBUGGG
+ #print("pipeline_run start", file = sys.stderr)
#
# default values
#
- if touch_files_only == False:
+ if touch_files_only is False:
touch_files_only = 0
- elif touch_files_only == True:
+ elif touch_files_only is True:
touch_files_only = 1
else:
touch_files_only = 2
# we are not running anything so do it as quickly as possible
one_second_per_job = False
- if verbose == None:
+ if verbose is None:
verbose = 1
- if verbose_abbreviated_path == None:
+ if verbose_abbreviated_path is None:
verbose_abbreviated_path = 2
# EXTRA pipeline_run DEBUGGING
@@ -3665,15 +5358,13 @@ def pipeline_run(target_tasks = [],
else:
EXTRA_PIPELINERUN_DEBUGGING = False
-
syncmanager = multiprocessing.Manager()
- if runtime_data == None:
+ if runtime_data is None:
runtime_data = {}
if not isinstance(runtime_data, dict):
- raise Exception("pipeline_run parameter runtime_data should be a dictionary of "
- "values passes to jobs at run time.")
-
+ raise Exception("pipeline_run parameter runtime_data should be a "
+ "dictionary of values passes to jobs at run time.")
#
# whether using multiprocessing or multithreading
@@ -3695,98 +5386,130 @@ def pipeline_run(target_tasks = [],
# Supplement mtime with system clock if using CHECKSUM_HISTORY_TIMESTAMPS
# we don't need to default to adding 1 second delays between jobs
#
- if one_second_per_job == None:
- if checksum_level == CHECKSUM_FILE_TIMESTAMPS:
- log_at_level (logger, 10, verbose, " Checksums rely on FILE TIMESTAMPS only and we don't know if the system file time resolution: Pause 1 second...")
+ if one_second_per_job is None:
+ if checksum_level == CHECKSUM_FILE_TIMESTAMPS:
+ log_at_level(logger, 10, verbose,
+ " Checksums rely on FILE TIMESTAMPS only and we don't know if the "
+ "system file time resolution: Pause 1 second...")
runtime_data["ONE_SECOND_PER_JOB"] = True
- else:
- log_at_level (logger, 10, verbose, " Checksum use calculated time as well: No 1 second pause...")
+ else:
+ log_at_level(logger, 10, verbose, " Checksum use calculated time as well: "
+ "No 1 second pause...")
runtime_data["ONE_SECOND_PER_JOB"] = False
else:
- log_at_level (logger, 10, verbose, " One second per job specified to be %s" % one_second_per_job)
+ log_at_level(logger, 10, verbose, " One second per job specified to be %s"
+ % one_second_per_job)
runtime_data["ONE_SECOND_PER_JOB"] = one_second_per_job
-
if verbose == 0:
logger = black_hole_logger
elif verbose >= 11:
# debugging aid: See t_stderr_logger
- # Each invocation of add_unique_prefix adds a unique prefix to all subsequent output
- # So that individual runs of pipeline run are tagged
+ # Each invocation of add_unique_prefix adds a unique prefix to
+ # all subsequent output So that individual runs of pipeline run
+ # are tagged
if hasattr(logger, "add_unique_prefix"):
logger.add_unique_prefix()
-
if touch_files_only and verbose >= 1:
logger.info("Touch output files instead of remaking them.")
- link_task_names_to_functions ()
- update_checksum_level_on_tasks (checksum_level)
-
#
- # If we aren't using checksums, and history file hasn't been specified,
- # we might be a bit surprised to find Ruffus writing to a sqlite db anyway.
- # Let us just use a in-memory db which will be thrown away
- # Of course, if history_file is specified, we presume you know what you are doing
+ # pipeline must be a Pipeline or a string naming a pipeline
#
- if checksum_level == CHECKSUM_FILE_TIMESTAMPS and history_file == None:
- history_file = ':memory:'
-
- job_history = open_job_history (history_file)
+ pipeline = lookup_pipeline(pipeline)
+ #
+ # Make sure all tasks in dependency list are linked to real functions
+ #
+ processed_tasks = set()
+ completed_pipeline_names = pipeline._complete_task_setup(processed_tasks)
+ # link_task_names_to_functions ()
+ update_checksum_level_on_tasks(checksum_level)
+ #
+ # If we aren't using checksums, and history file hasn't been specified,
+ # we might be a bit surprised to find Ruffus writing to a
+ # sqlite db anyway.
+ # Let us just dump to a placeholder memory db that can then be discarded
+ # Of course, if history_file is specified, we presume you know what
+ # you are doing
+ if checksum_level == CHECKSUM_FILE_TIMESTAMPS and history_file is None:
+ history_file = ':memory:'
+ job_history = open_job_history(history_file)
#
# @active_if decorated tasks can change their active state every time
# pipeline_run / pipeline_printout / pipeline_printout_graph is called
#
- update_active_states_for_all_tasks ()
-
+ update_active_states_for_all_tasks()
#
# target jobs
#
- target_tasks = task_names_to_tasks ("Target", target_tasks)
- forcedtorun_tasks = task_names_to_tasks ("Forced to run", forcedtorun_tasks)
+ target_tasks = lookup_tasks_from_user_specified_names("Target", target_tasks, pipeline.name)
+ if not target_tasks:
+ target_tasks = list(pipeline.tasks)
+ forcedtorun_tasks = lookup_tasks_from_user_specified_names("Forced to run", forcedtorun_tasks,
+ pipeline.name)
+
+ #
+ # forcedtorun_tasks and target_tasks may include more pipelines
+ # which have to be setup
+ #
+ incomplete_pipeline_names = set()
+ for task in forcedtorun_tasks + target_tasks:
+ if task.pipeline.name not in completed_pipeline_names:
+ incomplete_pipeline_names.add(task.pipeline.name)
+ for pipeline_name in incomplete_pipeline_names:
+ if pipeline_name in completed_pipeline_names:
+ continue
+ completed_pipeline_names = completed_pipeline_names.union(
+ pipeline.pipelines[pipeline_name]._complete_task_setup(processed_tasks))
#
- # To update the checksum file, we force all tasks to rerun but then don't actually call the task function...
+ # To update the checksum file, we force all tasks to rerun
+ # but then don't actually call the task function...
#
- # So starting with target_tasks and forcedtorun_tasks, we harvest all upstream dependencies willy, nilly
- # and assign the results to forcedtorun_tasks
+ # So starting with target_tasks and forcedtorun_tasks,
+ # we harvest all upstream dependencies willy, nilly
+ # and assign the results to forcedtorun_tasks
#
if touch_files_only == 2:
- (forcedtorun_tasks, ignore_param1, ignore_param2,
- ignore_param3) = topologically_sorted_nodes(target_tasks + forcedtorun_tasks, True,
- gnu_make_maximal_rebuild_mode,
- extra_data_for_signal = [t_verbose_logger(0, 0, None, runtime_data), job_history])
-
-
+ (forcedtorun_tasks, ignore_param1, ignore_param2, ignore_param3) = \
+ topologically_sorted_nodes(target_tasks + forcedtorun_tasks, True,
+ gnu_make_maximal_rebuild_mode,
+ extra_data_for_signal=[t_verbose_logger(0, 0, None,
+ runtime_data),
+ job_history],
+ signal_callback=is_node_up_to_date)
#
# If verbose >=10, for debugging:
# Prints which tasks trigger the pipeline rerun...
- # i.e. start from the farthest task, prints out all the up to date tasks, and the first out of date task
+ # i.e. start from the farthest task, prints out all the up to date
+ # tasks, and the first out of date task
#
- (incomplete_tasks,
- self_terminated_nodes,
- dag_violating_edges,
- dag_violating_nodes) = topologically_sorted_nodes( target_tasks, forcedtorun_tasks,
- gnu_make_maximal_rebuild_mode,
- extra_data_for_signal = [t_verbose_logger(verbose, verbose_abbreviated_path, logger, runtime_data), job_history])
+ (incomplete_tasks, self_terminated_nodes,
+ dag_violating_edges, dag_violating_nodes) = \
+ topologically_sorted_nodes(target_tasks, forcedtorun_tasks,
+ gnu_make_maximal_rebuild_mode,
+ extra_data_for_signal=[
+ t_verbose_logger(verbose, verbose_abbreviated_path,
+ logger, runtime_data),
+ job_history],
+ signal_callback=is_node_up_to_date)
if len(dag_violating_nodes):
dag_violating_tasks = ", ".join(t._name for t in dag_violating_nodes)
e = error_circular_dependencies("Circular dependencies found in the "
- "pipeline involving one or more of (%s)" %
- (dag_violating_tasks))
+ "pipeline involving one or more of "
+ "(%s)" % (dag_violating_tasks))
raise e
-
-
#
# get dependencies. Only include tasks which will be run
#
@@ -3794,34 +5517,36 @@ def pipeline_run(target_tasks = [],
task_parents = defaultdict(set)
for t in set_of_incomplete_tasks:
task_parents[t] = set()
- for parent in t._outward:
+ for parent in t._get_inward():
if parent in set_of_incomplete_tasks:
task_parents[t].add(parent)
-
#
# Print Complete tasks
#
# LOGGER level 5 : All jobs in All Tasks whether out of date or not
if verbose == 2 or verbose >= 5:
- (all_tasks, ignore_param1, ignore_param2,
- ignore_param3) = topologically_sorted_nodes(target_tasks, True,
- gnu_make_maximal_rebuild_mode,
- extra_data_for_signal = [t_verbose_logger(0, 0, None, runtime_data), job_history])
+ (all_tasks, ignore_param1, ignore_param2, ignore_param3) \
+ = topologically_sorted_nodes(target_tasks, True,
+ gnu_make_maximal_rebuild_mode,
+ extra_data_for_signal=[t_verbose_logger(0, 0, None,
+ runtime_data),
+ job_history],
+ signal_callback=is_node_up_to_date)
# indent hardcoded to 4
- for m in get_completed_task_strings (incomplete_tasks, all_tasks, forcedtorun_tasks, verbose, verbose_abbreviated_path, 4, runtime_data, job_history):
+ for m in get_completed_task_strings(incomplete_tasks, all_tasks,
+ forcedtorun_tasks, verbose,
+ verbose_abbreviated_path, 4,
+ runtime_data, job_history):
logger.info(m)
-
- #print json.dumps(task_parents.items(), indent=4, cls=task_encoder)
+ # print json.dumps(task_parents.items(), indent=4, cls=task_encoder)
logger.info("")
logger.info("_" * 40)
logger.info("Tasks which will be run:")
logger.info("")
logger.info("")
-
-
# prepare tasks for pipeline run:
#
# clear task outputs
@@ -3837,39 +5562,38 @@ def pipeline_run(target_tasks = [],
# BEWARE
# **********
for t in incomplete_tasks:
- t.init_for_pipeline()
-
+ t._init_for_pipeline()
#
# prime queue with initial set of job parameters
#
death_event = syncmanager.Event()
- #death_event = None
parameter_q = queue.Queue()
task_with_completed_job_q = queue.Queue()
- parameter_generator = make_job_parameter_generator (incomplete_tasks, task_parents,
- logger, forcedtorun_tasks,
- task_with_completed_job_q,
- runtime_data, verbose,
- verbose_abbreviated_path,
- syncmanager,
- death_event,
- touch_files_only, job_history)
+ parameter_generator = make_job_parameter_generator(incomplete_tasks, task_parents,
+ logger, forcedtorun_tasks,
+ task_with_completed_job_q,
+ runtime_data, verbose,
+ verbose_abbreviated_path,
+ syncmanager, death_event,
+ touch_files_only, job_history)
job_parameters = parameter_generator()
fill_queue_with_job_parameters(job_parameters, parameter_q, parallelism, logger, verbose)
#
# N.B.
# Handling keyboard shortcuts may require
- # See http://stackoverflow.com/questions/1408356/keyboard-interrupts-with-pythons-multiprocessing-pool
+ # See http://stackoverflow.com/questions/1408356/
+ # keyboard-interrupts-with-pythons-multiprocessing-pool
#
- # When waiting for a condition in threading.Condition.wait(), KeyboardInterrupt is never sent
+ # When waiting for a condition in threading.Condition.wait(),
+ # KeyboardInterrupt is never sent
# unless a timeout is specified
#
#
#
# #
- # # whether using multiprocessing
+ # whether using multiprocessing
# #
# pool = Pool(parallelism) if multiprocess > 1 else None
# if pool:
@@ -3883,7 +5607,8 @@ def pipeline_run(target_tasks = [],
# ....
#
#
- # it = pool_func(run_pooled_job_without_exceptions, feed_job_params_to_process_pool())
+ # it = pool_func(run_pooled_job_without_exceptions,
+ # feed_job_params_to_process_pool())
# while 1:
# try:
# job_result = it.next(*job_iterator_timeout)
@@ -3893,17 +5618,13 @@ def pipeline_run(target_tasks = [],
# except StopIteration:
# break
-
-
-
if pool:
pool_func = pool.imap_unordered
else:
pool_func = map
-
-
- feed_job_params_to_process_pool = feed_job_params_to_process_pool_factory (parameter_q, death_event, logger, verbose)
+ feed_job_params_to_process_pool = feed_job_params_to_process_pool_factory(
+ parameter_q, death_event, logger, verbose)
#
# for each result from job
@@ -3911,7 +5632,6 @@ def pipeline_run(target_tasks = [],
job_errors = RethrownJobError()
tasks_with_errors = set()
-
#
# job_result.job_name / job_result.return_value
# Reserved for returning result from job...
@@ -3920,41 +5640,35 @@ def pipeline_run(target_tasks = [],
# Rewrite for loop so we can call iter.next() with a timeout
try:
- #for job_result in pool_func(run_pooled_job_without_exceptions, feed_job_params_to_process_pool()):
- ii = iter(pool_func(run_pooled_job_without_exceptions, feed_job_params_to_process_pool()))
+ # for job_result in pool_func(run_pooled_job_without_exceptions,
+ # feed_job_params_to_process_pool()):
+ ii = iter(pool_func(run_pooled_job_without_exceptions, feed_job_params_to_process_pool()))
while 1:
- # Use a timeout of 3 years per job..., so that the condition we are waiting for in the thread
- # can be interrupted by signals... In other words, so that Ctrl-C works
- # Yucky part is that timeout is an extra parameter to IMapIterator.next(timeout=None)
- # but next() for normal iterators do not take any extra parameters.
+ # Use a timeout of 3 years per job..., so that the condition
+ # we are waiting for in the thread can be interrupted by
+ # signals... In other words, so that Ctrl-C works
+ # Yucky part is that timeout is an extra parameter to
+ # IMapIterator.next(timeout=None) but next() for normal
+ # iterators do not take any extra parameters.
if pool:
job_result = ii.next(timeout=99999999)
else:
job_result = next(ii)
# run next task
- log_at_level (logger, 11, verbose, "r" * 80 + "\n")
- t = node.lookup_node_from_name(job_result.task_name)
+ log_at_level(logger, 11, verbose, "r" * 80 + "\n")
+ t = node._lookup_node_from_index(job_result.node_index)
# remove failed jobs from history-- their output is bogus now!
if job_result.state in (JOB_ERROR, JOB_SIGNALLED_BREAK):
-
- if len(job_result.params) > 1: # some jobs have no outputs
- output_file_name = job_result.params[1]
- if not isinstance(output_file_name, list): # some have multiple outputs from one job
- output_file_name = [output_file_name]
- #
- # N.B. output parameters are not necessary all strings
- #
- for o_f_n in get_strings_in_nested_sequence(output_file_name):
- #
- # use paths relative to working directory
- #
- o_f_n = os.path.relpath(o_f_n)
- job_history.pop(o_f_n, None) # remove outfile from history if it exists
+ log_at_level(logger, 10, verbose, " JOB ERROR / JOB_SIGNALLED_BREAK: " + job_result.job_name)
+ # remove outfile from history if it exists
+ for o_f_n in get_job_result_output_file_names(job_result):
+ job_history.pop(o_f_n, None)
# only save poolsize number of errors
if job_result.state == JOB_ERROR:
- log_at_level (logger, 10, verbose, " Exception caught for %s" % job_result.job_name)
+ log_at_level(logger, 10, verbose, " Exception caught for %s"
+ % job_result.job_name)
job_errors.append(job_result.exception)
tasks_with_errors.add(t)
@@ -3962,33 +5676,36 @@ def pipeline_run(target_tasks = [],
# print to logger immediately
#
if log_exceptions:
- log_at_level (logger, 10, verbose, " Log Exception")
+ log_at_level(logger, 10, verbose, " Log Exception")
logger.error(job_errors.get_nth_exception_str())
#
# break if too many errors
#
if len(job_errors) >= parallelism or exceptions_terminate_immediately:
- log_at_level (logger, 10, verbose, " Break loop %s %s %s " % (exceptions_terminate_immediately, len(job_errors), parallelism) )
+ log_at_level(logger, 10, verbose, " Break loop %s %s %s "
+ % (exceptions_terminate_immediately,
+ len(job_errors), parallelism))
parameter_q.put(all_tasks_complete())
break
-
# break immediately if the user says stop
elif job_result.state == JOB_SIGNALLED_BREAK:
job_errors.append(job_result.exception)
job_errors.specify_task(t, "Exceptions running jobs")
- log_at_level (logger, 10, verbose, " Break loop JOB_SIGNALLED_BREAK %s %s " % (len(job_errors), parallelism) )
+ log_at_level(logger, 10, verbose, " Break loop JOB_SIGNALLED_BREAK %s %s "
+ % (len(job_errors), parallelism))
parameter_q.put(all_tasks_complete())
break
else:
if job_result.state == JOB_UP_TO_DATE:
# LOGGER: All Jobs in Out-of-date Tasks
- log_at_level (logger, 5, verbose, " %s unnecessary: already up to date" % job_result.job_name)
+ log_at_level(logger, 5, verbose, " %s unnecessary: already up to date"
+ % job_result.job_name)
else:
# LOGGER: Out-of-date Jobs in Out-of-date Tasks
- log_at_level (logger, 3, verbose, " %s completed" % job_result.job_name)
+ log_at_level(logger, 3, verbose, " %s completed" % job_result.job_name)
# save this task name and the job (input and output files)
# alternatively, we could just save the output file and its
# completion time, or on the other end of the spectrum,
@@ -4000,88 +5717,67 @@ def pipeline_run(target_tasks = [],
# chksum2 = md5.md5(marshal.dumps(t.user_defined_work_func.func_defaults) +
# marshal.dumps(t.args))
- if len(job_result.params) > 1: # some jobs have no outputs
- output_file_name = job_result.params[1]
- if not isinstance(output_file_name, list): # some have multiple outputs from one job
- output_file_name = [output_file_name]
- #
- # N.B. output parameters are not necessary all strings
- # and not all files have been successfully created,
- # even though the task apparently completed properly!
- # Remember to expand globs
- #
- for possible_glob_str in get_strings_in_nested_sequence(output_file_name):
- for o_f_n in glob.glob(possible_glob_str):
- #
- # use paths relative to working directory
- #
- o_f_n = os.path.relpath(o_f_n)
- try:
- log_at_level (logger, 10, verbose, " Job History for : " + o_f_n)
- mtime = os.path.getmtime(o_f_n)
- #
- # use probably higher resolution time.time() over mtime
- # which might have 1 or 2s resolutions, unless there is
- # clock skew and the filesystem time > system time
- # (e.g. for networks)
- #
- epoch_seconds = time.time()
- # Aargh. go back to insert one second between jobs
- if epoch_seconds < mtime:
- if one_second_per_job == None and not runtime_data["ONE_SECOND_PER_JOB"]:
- log_at_level (logger, 10, verbose, " Switch to one second per job")
- runtime_data["ONE_SECOND_PER_JOB"] = True
- elif epoch_seconds - mtime < 1.1:
- mtime = epoch_seconds
- chksum = JobHistoryChecksum(o_f_n, mtime, job_result.params[2:], t)
- job_history[o_f_n] = chksum
- except:
- pass
-
- ##for output_file_name in t.output_filenames:
- ## # could use current time instead...
- ## if not isinstance(output_file_name, list):
- ## output_file_name = [output_file_name]
- ## for o_f_n in output_file_name:
- ## mtime = os.path.getmtime(o_f_n)
- ## chksum = JobHistoryChecksum(o_f_n, mtime, job_result.params[2:], t)
- ## job_history[o_f_n] = chksum
-
-
- log_at_level (logger, 10, verbose, " signal completed task after checksumming...")
+ for o_f_n in get_job_result_output_file_names(job_result):
+ try:
+ log_at_level(logger, 10, verbose, " Job History : " + o_f_n)
+ mtime = os.path.getmtime(o_f_n)
+ #
+ # use probably higher resolution
+ # time.time() over mtime which might have 1 or 2s
+ # resolutions, unless there is clock skew and the
+ # filesystem time > system time (e.g. for networks)
+ #
+ epoch_seconds = time.time()
+ # Aargh. go back to insert one second between jobs
+ if epoch_seconds < mtime:
+ if one_second_per_job is None and \
+ not runtime_data["ONE_SECOND_PER_JOB"]:
+ log_at_level(logger, 10, verbose,
+ " Switch to 1s per job")
+ runtime_data["ONE_SECOND_PER_JOB"] = True
+ elif epoch_seconds - mtime < 1.1:
+ mtime = epoch_seconds
+ chksum = JobHistoryChecksum(o_f_n, mtime,
+ job_result.unglobbed_params[2:], t)
+ job_history[o_f_n] = chksum
+ log_at_level(logger, 10, verbose, " Job History Saved: " + o_f_n)
+ except:
+ pass
+
+ log_at_level(logger, 10, verbose, " _is_up_to_date completed task & checksum...")
#
- # signal completed task after checksumming
+ # _is_up_to_date completed task after checksumming
#
- task_with_completed_job_q.put((t, job_result.task_name, job_result.job_name))
-
+ task_with_completed_job_q.put((t,
+ job_result.task_name,
+ job_result.node_index,
+ job_result.job_name))
# make sure queue is still full after each job is retired
# do this after undating which jobs are incomplete
- log_at_level (logger, 10, verbose, " job errors?")
+ log_at_level(logger, 10, verbose, " job errors?")
if len(job_errors):
- #parameter_q.clear()
- #if len(job_errors) == 1 and not parameter_q._closed:
- log_at_level (logger, 10, verbose, " all tasks completed...")
+ # parameter_q.clear()
+ # if len(job_errors) == 1 and not parameter_q._closed:
+ log_at_level(logger, 10, verbose, " all tasks completed...")
parameter_q.put(all_tasks_complete())
else:
- log_at_level (logger, 10, verbose, " Fill queue with more parameter...")
- fill_queue_with_job_parameters(job_parameters, parameter_q, parallelism, logger, verbose)
+ log_at_level(logger, 10, verbose, " Fill queue with more parameter...")
+ fill_queue_with_job_parameters(job_parameters, parameter_q, parallelism, logger,
+ verbose)
# The equivalent of the normal end of a fall loop
except StopIteration as e:
- #print ("END iteration normally", file=sys.stderr)
pass
except:
exception_name, exception_value, exception_Traceback = sys.exc_info()
- exception_stack = traceback.format_exc()
+ exception_stack = traceback.format_exc()
# save exception to rethrow later
job_errors.append((None, None, exception_name, exception_value, exception_stack))
- log_at_level (logger, 10, verbose, " Exception caught %s" % (exception_value))
- log_at_level (logger, 10, verbose, " Exception caught %s" % (exception_name))
- log_at_level (logger, 10, verbose, " Exception caught %s" % (exception_stack))
- log_at_level (logger, 10, verbose, " Get next parameter size = %d" %
- parameter_q.qsize())
- log_at_level (logger, 10, verbose, " Task with completed jobs size = %d" %
- task_with_completed_job_q.qsize())
+ for ee in exception_value, exception_name, exception_stack:
+ log_at_level(logger, 10, verbose, " Exception caught %s" % (ee,))
+ log_at_level(logger, 10, verbose, " Get next parameter size = %d" % parameter_q.qsize())
+ log_at_level(logger, 10, verbose, " Task with completed "
+ "jobs size = %d" % task_with_completed_job_q.qsize())
parameter_q.put(all_tasks_complete())
try:
death_event.clear()
@@ -4089,55 +5785,51 @@ def pipeline_run(target_tasks = [],
pass
if pool:
- log_at_level (logger, 10, verbose, " pool.close")
+ log_at_level(logger, 10, verbose, " pool.close")
pool.close()
- log_at_level (logger, 10, verbose, " pool.terminate")
+ log_at_level(logger, 10, verbose, " pool.terminate")
try:
pool.terminate()
except:
pass
- log_at_level (logger, 10, verbose, " pool.terminated")
+ log_at_level(logger, 10, verbose, " pool.terminated")
raise job_errors
-
- #log_at_level (logger, 10, verbose, " syncmanager.shutdown")
- #syncmanager.shutdown()
-
+ # log_at_level (logger, 10, verbose, " syncmanager.shutdown")
+ # syncmanager.shutdown()
if pool:
- log_at_level (logger, 10, verbose, " pool.close")
- #pool.join()
+ log_at_level(logger, 10, verbose, " pool.close")
+ # pool.join()
pool.close()
- log_at_level (logger, 10, verbose, " pool.terminate")
+ log_at_level(logger, 10, verbose, " pool.terminate")
# an exception may be thrown after a signal is caught (Ctrl-C)
# when the EventProxy(s) for death_event might be left hanging
try:
pool.terminate()
except:
pass
- log_at_level (logger, 10, verbose, " pool.terminated")
+ log_at_level(logger, 10, verbose, " pool.terminated")
# Switch back off EXTRA pipeline_run DEBUGGING
EXTRA_PIPELINERUN_DEBUGGING = False
if len(job_errors):
raise job_errors
-
+ # DEBUGGG
+ #print("pipeline_run finish", file = sys.stderr)
# use high resolution timestamps where available
# default in python 2.5 and greater
-# N.B. File modify times / stat values have 1 second precision for many file systems
-# and may not be accurate to boot, especially over the network.
+# N.B. File modify times / stat values have 1 second precision for many file
+# systems and may not be accurate to boot, especially over the network.
os.stat_float_times(True)
if __name__ == '__main__':
import unittest
-
-
-
#
# debug parameter ignored if called as a module
#
diff --git a/ruffus/test/auto_generated_pipeline_examples/parallel.py b/ruffus/test/auto_generated_pipeline_examples/parallel.py
index b0f37e6..0bec4f1 100644
--- a/ruffus/test/auto_generated_pipeline_examples/parallel.py
+++ b/ruffus/test/auto_generated_pipeline_examples/parallel.py
@@ -146,7 +146,7 @@ def test_job_io(infiles, outfiles, extra_params):
if isinstance(infiles, str):
infiles = [infiles]
- elif infiles == None:
+ elif infiles is None:
infiles = []
if isinstance(outfiles, str):
outfiles = [outfiles]
@@ -225,4 +225,4 @@ elif options.dependency_file:
options.forced_tasks)
else:
pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs)
-
+
diff --git a/ruffus/test/auto_generated_pipeline_examples/simple.py b/ruffus/test/auto_generated_pipeline_examples/simple.py
index 1a71286..361969d 100644
--- a/ruffus/test/auto_generated_pipeline_examples/simple.py
+++ b/ruffus/test/auto_generated_pipeline_examples/simple.py
@@ -146,7 +146,7 @@ def test_job_io(infiles, outfiles, extra_params):
if isinstance(infiles, str):
infiles = [infiles]
- elif infiles == None:
+ elif infiles is None:
infiles = []
if isinstance(outfiles, str):
outfiles = [outfiles]
@@ -250,4 +250,4 @@ elif options.dependency_file:
options.forced_tasks)
else:
pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs)
-
+
diff --git a/ruffus/test/auto_generated_pipeline_examples/simpler.py b/ruffus/test/auto_generated_pipeline_examples/simpler.py
index 3420fcf..29ee6dc 100644
--- a/ruffus/test/auto_generated_pipeline_examples/simpler.py
+++ b/ruffus/test/auto_generated_pipeline_examples/simpler.py
@@ -156,7 +156,7 @@ def test_job_io(infiles, outfiles, extra_params):
if isinstance(infiles, str):
infiles = [infiles]
- elif infiles == None:
+ elif infiles is None:
infiles = []
if isinstance(outfiles, str):
outfiles = [outfiles]
@@ -266,4 +266,4 @@ elif options.dependency_file:
else:
pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
-
+
diff --git a/ruffus/test/create_test_script_from_dependency_tree.py b/ruffus/test/create_test_script_from_dependency_tree.py
index 96712fa..2a521bc 100755
--- a/ruffus/test/create_test_script_from_dependency_tree.py
+++ b/ruffus/test/create_test_script_from_dependency_tree.py
@@ -9,7 +9,7 @@ from __future__ import print_function
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-# options
+# options
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -39,13 +39,13 @@ from ruffus import *
parser = OptionParser(version="%prog 1.0")
parser.add_option("-i", "--input_dot_file", dest="dot_file",
- metavar="FILE",
+ metavar="FILE",
default = os.path.join(exe_path, "dependency_data", "simple.dag"),
type="string",
help="name and path of tree file in modified DOT format used to generate "
"test script dependencies.")
parser.add_option("-o", "--output_file", dest="output_file",
- metavar="FILE",
+ metavar="FILE",
default = os.path.join(exe_path, "pipelines", "simple.py"),
type="string",
help="name and path of output python test script.")
@@ -56,7 +56,7 @@ parser.add_option( "-J", "--jumble_task_order", dest = "jumble_task_order",
action="store_true", default=False,
help="Do not define task functions in order of dependency.")
-parameters = [
+parameters = [
]
mandatory_parameters = ["dot_file"]
@@ -68,7 +68,7 @@ mandatory_parameters = ["dot_file"]
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-# imports
+# imports
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -86,7 +86,7 @@ try:
except ImportError:
import simplejson
json = simplejson
-
+
dumps = json.dumps
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -120,29 +120,29 @@ def task_dependencies_from_dotfile(stream):
decorator_regex = re.compile(r"([a-z_]+) *(\(.*)")
attributes_regex = re.compile(r"\[.*\]")
-
+
which_task_follows = defaultdict(list)
task_decorators = defaultdict(list)
task_descriptions = dict()
#
# remember node
- #
+ #
all_tasks = dict()
io_tasks = set()
for linenum, line in enumerate(stream):
# remove heading and trailing spaces
line = line.strip()
-
+
if "digraph" in line:
continue;
if not len(line):
continue
-
+
#
# decorators
- #
+ #
if line[0:2] == '#@':
fields = line[2:].split('::', 2)
if len(fields) != 3:
@@ -153,7 +153,7 @@ def task_dependencies_from_dotfile(stream):
raise Exception("Task decorator missing starting ampersand '@' on line# %d\n(%s)" %
(linenum, line))
for d in decorators[1:].split("@"):
- m = decorator_regex.match(d)
+ m = decorator_regex.match(d)
if not m:
raise Exception("Task decorator (%s) missing parentheses on line# %d\n(%s)" %
(d, linenum, line))
@@ -163,15 +163,15 @@ def task_dependencies_from_dotfile(stream):
task_descriptions[task_name] = description
continue
-
+
#
# other comments
- #
+ #
if line[0] in '#{}/':
continue;
line = line.strip(';')
line = attributes_regex.sub("", line)
-
+
#
# ignore assignments
#
@@ -182,12 +182,12 @@ def task_dependencies_from_dotfile(stream):
which_task_follows[name2].append(name1)
all_tasks[name1] = 1
all_tasks[name2] = 1
-
+
for task in task_decorators:
if task not in all_tasks:
raise Exception("Decorated task %s not in dependencies")
-
-
+
+
# order tasks by precedence the dump way: iterating until true
disordered = True
while (disordered):
@@ -197,7 +197,7 @@ def task_dependencies_from_dotfile(stream):
if all_tasks[to_task] <= all_tasks[f]:
all_tasks[to_task] += all_tasks[f]
disordered = True
-
+
sorted_task_names = list(sorted(list(all_tasks.keys()), key=lambda x:all_tasks[x]))
return which_task_follows, sorted_task_names, task_decorators, io_tasks, task_descriptions
@@ -209,7 +209,7 @@ def task_dependencies_from_dotfile(stream):
# generate_program_task_file
#_________________________________________________________________________________________
-def generate_program_task_file(stream, task_dependencies, task_names,
+def generate_program_task_file(stream, task_dependencies, task_names,
task_decorators, io_tasks, task_descriptions):
print("task_decorators = ", dumps(task_decorators, indent = 4), file=sys.stderr)
@@ -223,7 +223,7 @@ def generate_program_task_file(stream, task_dependencies, task_names,
defined_tasks = set()
#
- # iterate through tasks
+ # iterate through tasks
#
for task_name in task_names:
defined_tasks.add(task_name)
@@ -236,14 +236,14 @@ def generate_program_task_file(stream, task_dependencies, task_names,
stream.write("@" + decorator + decorator_parameters + "\n")
#
- # write task dependencies
+ # write task dependencies
#
if task_name in task_dependencies:
- params = ", ".join(t if t in defined_tasks else '"%s"' % t
+ params = ", ".join(t if t in defined_tasks else '"%s"' % t
for t in task_dependencies[task_name])
stream.write("@follows(%s)\n" % params)
-
+
#
# Function body
#
@@ -255,40 +255,40 @@ def generate_program_task_file(stream, task_dependencies, task_names,
description = description.replace("\n", " \n")
stream.write(' %s\n' % description)
stream.write(' """\n')
-
+
stream.write(" test_job_io(infiles, outfiles, extra_params)\n")
#else:
# stream.write("def %s(*params):\n" % task_name)
-
+
stream.write("\n\n")
stream.write(
"""
-#
+#
# Necessary to protect the "entry point" of the program under windows.
# see: http://docs.python.org/library/multiprocessing.html#multiprocessing-programming
#
if __name__ == '__main__':
try:
if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
- long_winded=True,
+ pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
+ long_winded=True,
gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
-
+
elif options.dependency_file:
pipeline_printout_graph ( open(options.dependency_file, "w"),
options.dependency_graph_format,
- options.target_tasks,
+ options.target_tasks,
options.forced_tasks,
draw_vertically = not options.draw_horizontally,
gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
no_key_legend = options.no_key_legend_in_graph)
- else:
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
+ else:
+ pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
except Exception, e:
- print e.args
+ print e.args
\n""")
@@ -311,12 +311,12 @@ helpstr = f.getvalue()
(options, remaining_args) = parser.parse_args()
# mandatory options
for parameter in mandatory_parameters:
- if options.__dict__[parameter] == None:
+ if options.__dict__[parameter] is None:
die_error("Please specify a file in --%s.\n\n" % parameter + helpstr)
-(task_dependencies, task_names,
+(task_dependencies, task_names,
task_decorators, io_tasks,
task_descriptions) = task_dependencies_from_dotfile(open(options.dot_file))
@@ -324,9 +324,9 @@ output_file = open(options.output_file, "w")
#
# print template for python file output
-#
+#
output_file.write(open(os.path.join(exe_path,"test_script.py_template")).read().
- replace("PYPER_PATH",
+ replace("PYPER_PATH",
os.path.abspath(os.path.join(exe_path, "..", ".."))))
-generate_program_task_file(output_file, task_dependencies, task_names,
+generate_program_task_file(output_file, task_dependencies, task_names,
task_decorators, io_tasks, task_descriptions)
diff --git a/ruffus/test/draw_specified_dependency_tree.py b/ruffus/test/draw_specified_dependency_tree.py
index 009216b..5f31f08 100755
--- a/ruffus/test/draw_specified_dependency_tree.py
+++ b/ruffus/test/draw_specified_dependency_tree.py
@@ -9,7 +9,7 @@ from __future__ import print_function
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-# options
+# options
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -34,26 +34,26 @@ else:
parser = OptionParser(version="%prog 1.0")
parser.add_option("-d", "--dot_file", dest="dot_file",
- metavar="FILE",
+ metavar="FILE",
default = os.path.join(exe_path, "test_data/dag.dependency"),
type="string",
help="name and path of tree file in DOT format")
parser.add_option("-u", "--uptodate_job_names", dest="uptodate_job_names",
action="append",
- metavar="JOBNAME",
+ metavar="JOBNAME",
default = list(),
type="string",
help="nodes to terminate on.")
parser.add_option("-t", "--target_job_names", dest="target_job_names",
action="append",
default = list(),
- metavar="JOBNAME",
+ metavar="JOBNAME",
type="string",
help="nodes to start on.")
parser.add_option("-f", "--forced_job_names", dest="forced_job_names",
action="append",
default = list(),
- metavar="JOBNAME",
+ metavar="JOBNAME",
type="string",
help="nodes to start on.")
parser.add_option("-v", "--verbose", dest = "verbose",
@@ -69,7 +69,7 @@ parser.add_option("--skip_up_to_date", dest="skip_up_to_date",
action="store_true", default=False,
help="Only draw tasks which need to be rerun")
-parameters = [
+parameters = [
"uptodate_job_names"
]
@@ -82,7 +82,7 @@ mandatory_parameters = ["dot_file"]
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-# imports
+# imports
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -116,7 +116,7 @@ def make_tree_from_dotfile (stream):
#
# remember node
- #
+ #
for linenum, line in enumerate(stream):
@@ -131,16 +131,16 @@ def make_tree_from_dotfile (stream):
continue;
nodes = [x.strip() for x in line.split('->')]
for name1, name2 in adjacent_pairs_iterate(nodes):
- if not node.is_node(name1):
+ if not node._is_node(name1):
node(name1)
- if not node.is_node(name2):
+ if not node._is_node(name2):
node(name2)
- node.lookup_node_from_name(name2).add_child(node.lookup_node_from_name(name1))
+ node._lookup_node_from_name(name2).add_child(node._lookup_node_from_name(name1))
#
# task hack
- node.lookup_node_from_name(name2)._action = 1
- node.lookup_node_from_name(name1)._action = 1
+ node._lookup_node_from_name(name2)._action = 1
+ node._lookup_node_from_name(name1)._action = 1
@@ -180,7 +180,7 @@ if __name__ == '__main__':
(options, remaining_args) = parser.parse_args()
# mandatory options
for parameter in mandatory_parameters:
- if options.__dict__[parameter] == None:
+ if options.__dict__[parameter] is None:
die_error("Please specify a file in --%s.\n\n" % parameter + helpstr)
@@ -192,17 +192,17 @@ if __name__ == '__main__':
#
# set up_to_date jobs
- #
+ #
uptodate_jobs = task_names_to_tasks ("Up to date", options.uptodate_job_names)
for n in uptodate_jobs:
n._signal = True
- graph_printout_in_dot_format (sys.stdout,
- options.target_job_names,
- options.forced_job_names,
- not options.horizontal_graph,
+ graph_printout_in_dot_format (sys.stdout,
+ options.target_job_names,
+ options.forced_job_names,
+ not options.horizontal_graph,
options.skip_upstream,
options.skip_up_to_date)
-
+
diff --git a/ruffus/test/test_ctrl_c_exceptions.py b/ruffus/test/manual_test_ctrl_c_exceptions.py
similarity index 100%
rename from ruffus/test/test_ctrl_c_exceptions.py
rename to ruffus/test/manual_test_ctrl_c_exceptions.py
diff --git a/ruffus/test/test_drmaa.py b/ruffus/test/manual_test_drmaa.py
similarity index 100%
rename from ruffus/test/test_drmaa.py
rename to ruffus/test/manual_test_drmaa.py
diff --git a/ruffus/test/play_with_colours.py b/ruffus/test/play_with_colours.py
index 30c7b3c..ca51ea0 100755
--- a/ruffus/test/play_with_colours.py
+++ b/ruffus/test/play_with_colours.py
@@ -210,7 +210,7 @@ custom_flow_chart_colour_scheme["colour_scheme_index"] = options.colour_scheme_i
#
# Overriding colours
#
-if options.colour_scheme_index == None:
+if options.colour_scheme_index is None:
custom_flow_chart_colour_scheme["Vicious cycle"]["linecolor"] = '"#FF3232"'
custom_flow_chart_colour_scheme["Pipeline"]["fontcolor"] = '"#FF3232"'
custom_flow_chart_colour_scheme["Key"]["fontcolor"] = "black"
diff --git a/ruffus/test/simpler_at_runtime.py b/ruffus/test/simpler_at_runtime.py
deleted file mode 100755
index 9c79bdf..0000000
--- a/ruffus/test/simpler_at_runtime.py
+++ /dev/null
@@ -1,253 +0,0 @@
-#!/usr/bin/env python
-from __future__ import print_function
-"""
-
- test_tasks.py
-
-"""
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# options
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-from optparse import OptionParser
-import sys, os
-import os.path
-try:
- import StringIO as io
-except:
- import io as io
-
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-if __name__ == '__main__':
- module_name = os.path.split(sys.argv[0])[1]
- module_name = os.path.splitext(module_name)[0];
-else:
- module_name = __name__
-
-
-
-
-parser = OptionParser(version="%prog 1.0")
-parser.add_option("-t", "--target_tasks", dest="target_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Target task(s) of pipeline.")
-parser.add_option("-R", "--runtime_files", dest="runtime_files",
- action="append",
- default = ["a.3"],
- metavar="JOBNAME",
- type="string",
- help="Target task(s) of pipeline.")
-parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Pipeline task(s) which will be included even if they are up to date.")
-parser.add_option("-j", "--jobs", dest="jobs",
- default=1,
- metavar="jobs",
- type="int",
- help="Specifies the number of jobs (commands) to run simultaneously.")
-parser.add_option("-v", "--verbose", dest = "verbose",
- action="count", default=0,
- help="Print more verbose messages for each additional verbose level.")
-parser.add_option("-d", "--debug", dest = "debug",
- action="count", default=0,
- help="Cleanup afterwards.")
-parser.add_option("--dependency", dest="dependency_file",
- metavar="FILE",
- type="string",
- help="Print a dependency graph of the pipeline that would be executed "
- "to FILE, but do not execute it.")
-parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
- metavar="FORMAT",
- type="string",
- default = 'svg',
- help="format of dependency graph file. Can be 'ps' (PostScript), "+
- "'svg' 'svgz' (Structured Vector Graphics), " +
- "'png' 'gif' (bitmap graphics) etc ")
-parser.add_option("-n", "--just_print", dest="just_print",
- action="store_true", default=False,
- help="Print a description of the jobs that would be executed, "
- "but do not execute them.")
-parser.add_option("-M", "--minimal_rebuild_mode", dest="minimal_rebuild_mode",
- action="store_true", default=False,
- help="Rebuild a minimum of tasks necessary for the target. "
- "Ignore upstream out of date tasks if intervening tasks are fine.")
-parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
- action="store_true", default=False,
- help="Do not print out legend and key for dependency graph.")
-parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
- action="store_true", default=False,
- help="Draw horizontal dependency graph.")
-
-parameters = [
- ]
-
-
-
-
-
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# imports
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-import re
-import operator
-import sys,os
-from collections import defaultdict
-
-sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-
-# use simplejson in place of json for python < 2.6
-try:
- import json
-except ImportError:
- import simplejson
- json = simplejson
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# Functions
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# Main logic
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-
-
-
-
-# get help string
-f =io.StringIO()
-parser.print_help(f)
-helpstr = f.getvalue()
-(options, remaining_args) = parser.parse_args()
-
-
-
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# Tasks
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-
-#
-# task1
-#
- at originate(['a.1'] + options.runtime_files)
-def task1(outfile):
- """
- First task
- """
- output_text = ""
- output_text += " -> " + json.dumps(outfile) + "\n"
- open(outfile, "w").write(output_text)
-
-
-
-#
-# task2
-#
- at transform(task1, suffix(".1"), ".2")
-def task2(infile, outfile):
- """
- Second task
- """
- output_text = open(infile).read() if infile else ""
- output_text += json.dumps(infile) + " -> " + json.dumps(outfile) + "\n"
- open(outfile, "w").write(output_text)
-
-
-
-#
-# task3
-#
- at transform(task2, suffix(".2"), ".3")
-def task3(infile, outfile):
- """
- Third task
- """
- output_text = open(infile).read() if infile else ""
- output_text += json.dumps(infile) + " -> " + json.dumps(outfile) + "\n"
- open(outfile, "w").write(output_text)
-
-
-
-#
-# task4
-#
- at follows(task3)
- at transform(runtime_parameter("a"), suffix(".3"), ".4")
-def task4(infile, outfile):
- """
- Fourth task
- """
- output_text = open(infile).read() if infile else ""
- output_text += json.dumps(infile) + " -> " + json.dumps(outfile) + "\n"
- open(outfile, "w").write(output_text)
-
-#
-# Necessary to protect the "entry point" of the program under windows.
-# see: http://docs.python.org/library/multiprocessing.html#multiprocessing-programming
-#
-if __name__ == '__main__':
- if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- verbose = options.verbose, runtime_data = {"a": options.runtime_files})
-
- elif options.dependency_file:
- pipeline_printout_graph ( open(options.dependency_file, "w"),
- options.dependency_graph_format,
- options.target_tasks,
- options.forced_tasks,
- draw_vertically = not options.draw_horizontally,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- no_key_legend = options.no_key_legend_in_graph)
- elif options.debug:
- import os
- for f in ["a.1", "a.2","a.3","a.4"]:
- if os.path.exists(f):
- os.unlink(f)
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- verbose = options.verbose, runtime_data = {"a": options.runtime_files})
- for f in ["a.1", "a.2","a.3","a.4"]:
- if os.path.exists(f):
- os.unlink(f)
- else:
- raise Exception("%s is missing" % f)
- print("OK")
- else:
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- verbose = options.verbose, runtime_data = {"a": options.runtime_files})
diff --git a/ruffus/test/simpler_with_shared_logging.py b/ruffus/test/simpler_with_shared_logging.py
index 0ef6545..f35ccc0 100755
--- a/ruffus/test/simpler_with_shared_logging.py
+++ b/ruffus/test/simpler_with_shared_logging.py
@@ -181,7 +181,7 @@ def test_job_io(infiles, outfiles, extra_params):
if isinstance(infiles, str):
infiles = [infiles]
- elif infiles == None:
+ elif infiles is None:
infiles = []
if isinstance(outfiles, str):
outfiles = [outfiles]
diff --git a/ruffus/test/test_N_x_M_and_collate.py b/ruffus/test/test_N_x_M_and_collate.py
index 3f3b3ab..0262904 100755
--- a/ruffus/test/test_N_x_M_and_collate.py
+++ b/ruffus/test/test_N_x_M_and_collate.py
@@ -46,88 +46,36 @@ CNT_SIMULATION_FILES = 3
-import os, sys
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0, os.path.abspath(os.path.join(exe_path,"..", "..")))
-
-
-
-from ruffus import *
-import random
-
-
+import os
+import sys
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
-# options
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
-from optparse import OptionParser
-parser = OptionParser(version="%prog 1.0")
-parser.add_option("-D", "--debug", dest = "debug",
- action="store_true", default=False,
- help="Run as unit test with default values.")
-parser.add_option("-k", "--keep", dest = "keep",
- action="store_true", default=False,
- help="Do not cleanup after unit test runs.")
-parser.add_option("-t", "--target_tasks", dest="target_tasks",
- action="append",
- default = ["statistical_summary"],
- metavar="JOBNAME",
- type="string",
- help="Target task(s) of pipeline.")
-parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Pipeline task(s) which will be included even if they are up to date.")
-parser.add_option("-j", "--jobs", dest="jobs",
- default=5,
- metavar="jobs",
- type="int",
- help="Specifies the number of jobs (commands) to run simultaneously.")
-
-parser.add_option("-g", "--gene_data_dir", dest="gene_data_dir",
- default="%s/temp_gene_data_for_intermediate_example" % exe_path,
- metavar="PATH",
- type="string",
- help="Directory with gene data [*.genes / *.gwas].")
-parser.add_option("-s", "--simulation_data_dir", dest="simulation_data_dir",
- default="%s/temp_simulation_data_for_intermediate_example" % exe_path,
- metavar="PATH",
- type="string",
- help="Directory with simulation data [*.simulation].")
-parser.add_option("-w", "--working_dir", dest="working_dir",
- default="%s/working_dir_for_intermediate_example" % exe_path,
- metavar="PATH",
- type="string",
- help="Working directory.")
-
-
-parser.add_option("-v", "--verbose", dest = "verbose",
- action="count", default=0,
- help="Print more verbose messages for each additional verbose level.")
-parser.add_option("-d", "--dependency", dest="dependency_file",
- metavar="FILE",
- type="string",
- help="Print a dependency graph of the pipeline that would be executed "
- "to FILE, but do not execute it.")
-parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
- metavar="FORMAT",
- type="string",
- default = 'svg',
- help="format of dependency graph file. Can be 'ps' (PostScript), "+
- "'svg' 'svgz' (Structured Vector Graphics), " +
- "'png' 'gif' (bitmap graphics) etc ")
-parser.add_option("-n", "--just_print", dest="just_print",
- action="store_true", default=False,
- help="Print a description of the jobs that would be executed, "
- "but do not execute them.")
+try:
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
+import random
+script_path = os.path.dirname(os.path.abspath(__file__))
+gene_data_dir ="%s/temp_gene_data_for_intermediate_example" % script_path
+simulation_data_dir = "%s/temp_simulation_data_for_intermediate_example" % script_path
+working_dir = "%s/working_dir_for_intermediate_example" % script_path
@@ -171,14 +119,14 @@ def get_gene_gwas_file_pairs( ):
"""
- gene_files = glob.glob(os.path.join(options.gene_data_dir, "*.gene"))
- gwas_files = glob.glob(os.path.join(options.gene_data_dir, "*.gwas"))
+ gene_files = glob.glob(os.path.join(gene_data_dir, "*.gene"))
+ gwas_files = glob.glob(os.path.join(gene_data_dir, "*.gwas"))
common_roots = set([os.path.splitext(os.path.split(x)[1])[0] for x in gene_files])
common_roots &=set([os.path.splitext(os.path.split(x)[1])[0] for x in gwas_files])
common_roots = list(common_roots)
- p = os.path; g_dir = options.gene_data_dir
+ p = os.path; g_dir = gene_data_dir
file_pairs = [[p.join(g_dir, x + ".gene"), p.join(g_dir, x + ".gwas")] for x in common_roots]
@@ -197,7 +145,7 @@ def get_simulation_files( ):
file with .simulation extensions,
corresponding roots (no extension) of each file
"""
- simulation_files = glob.glob(os.path.join(options.simulation_data_dir, "*.simulation"))
+ simulation_files = glob.glob(os.path.join(simulation_data_dir, "*.simulation"))
simulation_roots =[os.path.splitext(os.path.split(x)[1])[0] for x in simulation_files]
return simulation_files, simulation_roots
@@ -214,14 +162,6 @@ def get_simulation_files( ):
-# get help string
-f =io.StringIO()
-parser.print_help(f)
-helpstr = f.getvalue()
-(options, remaining_args) = parser.parse_args()
-
-
-working_dir = options.working_dir
@@ -235,22 +175,22 @@ working_dir = options.working_dir
#
# mkdir: makes sure output directories exist before task
#
- at follows(mkdir(options.gene_data_dir, options.simulation_data_dir))
+ at follows(mkdir(gene_data_dir, simulation_data_dir))
def setup_simulation_data ():
"""
create simulation files
"""
for i in range(CNT_GENE_GWAS_FILES):
- open(os.path.join(options.gene_data_dir, "%03d.gene" % i), "w")
- open(os.path.join(options.gene_data_dir, "%03d.gwas" % i), "w")
+ open(os.path.join(gene_data_dir, "%03d.gene" % i), "w").close()
+ open(os.path.join(gene_data_dir, "%03d.gwas" % i), "w").close()
# gene files without corresponding gwas and vice versa
- open(os.path.join(options.gene_data_dir, "orphan1.gene"), "w")
- open(os.path.join(options.gene_data_dir, "orphan2.gwas"), "w")
- open(os.path.join(options.gene_data_dir, "orphan3.gwas"), "w")
+ open(os.path.join(gene_data_dir, "orphan1.gene"), "w").close()
+ open(os.path.join(gene_data_dir, "orphan2.gwas"), "w").close()
+ open(os.path.join(gene_data_dir, "orphan3.gwas"), "w").close()
for i in range(CNT_SIMULATION_FILES):
- open(os.path.join(options.simulation_data_dir, "%03d.simulation" % i), "w")
+ open(os.path.join(simulation_data_dir, "%03d.simulation" % i), "w").close()
@@ -271,24 +211,22 @@ def cleanup_simulation_data ():
"""
cleanup files
"""
- if options.verbose:
- sys.stderr.write("Cleanup working directory and simulation files.\n")
#
# cleanup gene and gwas files
#
- for f in glob.glob(os.path.join(options.gene_data_dir, "*.gene")):
+ for f in glob.glob(os.path.join(gene_data_dir, "*.gene")):
os.unlink(f)
- for f in glob.glob(os.path.join(options.gene_data_dir, "*.gwas")):
+ for f in glob.glob(os.path.join(gene_data_dir, "*.gwas")):
os.unlink(f)
- try_rmdir(options.gene_data_dir)
+ try_rmdir(gene_data_dir)
#
# cleanup simulation
#
- for f in glob.glob(os.path.join(options.simulation_data_dir, "*.simulation")):
+ for f in glob.glob(os.path.join(simulation_data_dir, "*.simulation")):
os.unlink(f)
- try_rmdir(options.simulation_data_dir)
+ try_rmdir(simulation_data_dir)
#
@@ -335,7 +273,8 @@ def generate_simulation_params ():
#
# mkdir: makes sure output directories exist before task
#
- at follows(mkdir(options.working_dir, os.path.join(working_dir, "simulation_results")))
+ at follows(setup_simulation_data)
+ at follows(mkdir(working_dir, os.path.join(working_dir, "simulation_results")))
@files(generate_simulation_params)
def gwas_simulation(input_files, result_file_path, gene_file_root, sim_file_root, result_file):
"""
@@ -348,6 +287,7 @@ def gwas_simulation(input_files, result_file_path, gene_file_root, sim_file_root
simulation_res_file = open(result_file_path, "w")
simulation_res_file.write("%s + %s -> %s\n" % (gene_file_root, sim_file_root, result_file))
+ simulation_res_file.close()
#_________________________________________________________________________________________
@@ -372,54 +312,55 @@ def statistical_summary (result_files, summary_file):
summary_file = open(summary_file, "w")
for f in result_files:
- summary_file.write(open(f).read())
+ with open(f) as ii:
+ summary_file.write(ii.read())
+ summary_file.close()
+
+import unittest, shutil
+try:
+ from StringIO import StringIO
+except:
+ from io import StringIO
+class Test_ruffus(unittest.TestCase):
+ def tearDown(self):
+ try:
+ cleanup_simulation_data ()
+ pass
+ except:
+ pass
+ def test_ruffus (self):
+ pipeline_run(multiprocess = 50, verbose = 0)
+ for oo in "000.mean", "001.mean":
+ results_file_name = os.path.join(working_dir, oo)
+ if not os.path.exists(results_file_name):
+ raise Exception("Missing %s" % results_file_name)
+ def test_newstyle_ruffus (self):
-#888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-#
-# print pipeline or run pipeline
-#
+ test_pipeline = Pipeline("test")
-#
-# Necessary to protect the "entry point" of the program under windows.
-# see: http://docs.python.org/library/multiprocessing.html#multiprocessing-programming
-#
-if __name__ == '__main__':
- try:
- if options.debug:
- if not len(options.target_tasks):
- options.target_tasks.append([statistical_summary])
- pipeline_run([setup_simulation_data], [setup_simulation_data], multiprocess = options.jobs, verbose = 0)
- else:
- if (not len(get_gene_gwas_file_pairs( )[0]) or
- not len (get_simulation_files( )[0])):
- print("Warning!!\n\n\tNo *.gene / *.gwas or *.simulation: Run --debug to create simulation files first\n\n")
- sys.exit(1)
-
-
- if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks, verbose=options.verbose)
-
- elif options.dependency_file:
- graph_printout ( open(options.dependency_file, "w"),
- options.dependency_graph_format,
- options.target_tasks,
- options.forced_tasks)
- else:
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs, verbose = options.verbose)
-
-
- if options.debug and not options.keep:
- cleanup_simulation_data ()
+ test_pipeline.follows(setup_simulation_data, mkdir(gene_data_dir, simulation_data_dir))
+
+ test_pipeline.files(gwas_simulation, generate_simulation_params)\
+ .follows(setup_simulation_data)\
+ .follows(mkdir(working_dir, os.path.join(working_dir, "simulation_results")))
- except Exception as e:
- print(e.args)
- raise
+ test_pipeline.collate(statistical_summary, gwas_simulation, regex(r"simulation_results/(\d+).\d+.simulation_res"), r"\1.mean")\
+ .posttask(lambda : sys.stdout.write("\nOK\n"))
+
+ test_pipeline.run(multiprocess = 50, verbose = 0)
+ for oo in "000.mean", "001.mean":
+ results_file_name = os.path.join(working_dir, oo)
+ if not os.path.exists(results_file_name):
+ raise Exception("Missing %s" % results_file_name)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/ruffus/test/test_active_if.py b/ruffus/test/test_active_if.py
index d8435b9..08b289f 100755
--- a/ruffus/test/test_active_if.py
+++ b/ruffus/test/test_active_if.py
@@ -7,42 +7,33 @@ from __future__ import print_function
"""
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# options
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+tempdir = "test_active_if"
-from optparse import OptionParser
-import sys, os
-import os.path
-try:
- import StringIO as io
-except:
- import io as io
-
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-# add self to search path for testing
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-if __name__ == '__main__':
- module_name = os.path.split(sys.argv[0])[1]
- module_name = os.path.splitext(module_name)[0];
-else:
- module_name = __name__
+import os
+import sys
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
-from ruffus import *
-parser = cmdline.get_argparse( description='Test @active_if')
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+try:
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
-options = parser.parse_args()
-
-# optional logger which can be passed to ruffus tasks
-logger, logger_mutex = cmdline.setup_logging (__name__, options.log_file, options.verbose)
@@ -57,22 +48,14 @@ logger, logger_mutex = cmdline.setup_logging (__name__, options.log_file, option
import re
import operator
-import sys,os
from collections import defaultdict
+import unittest, shutil
import json
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-# Functions
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
# Main logic
@@ -93,15 +76,17 @@ def helper (infiles, outfiles):
preamble_len = 0
for infile in infiles:
if infile:
- for line in open(infile):
- output_text += line
- preamble_len = max(preamble_len, len(line) - len(line.lstrip()))
+ with open(infile) as ii:
+ for line in ii:
+ output_text += line
+ preamble_len = max(preamble_len, len(line) - len(line.lstrip()))
preamble = " " * (preamble_len + 4) if len(output_text) else ""
for outfile in outfiles:
file_output_text = preamble + json.dumps(infile) + " -> " + json.dumps(outfile) + "\n"
- open(outfile, "w").write(output_text + file_output_text)
+ with open(outfile, "w") as oo:
+ oo.write(output_text + file_output_text)
@@ -122,7 +107,7 @@ def task1(outfile, extra):
"""
First task
"""
- sys.stderr.write("originate works with outfile '%s'" % outfile + " and " + extra + "\n")
+ # N.B. originate works with an extra parameter
helper (None, outfile)
@@ -194,25 +179,81 @@ null -> "test_active_if/b.1"
"test_active_if/b.4" -> "test_active_if/summary.5"
"""
-#
-# Necessary to protect the "entry point" of the program under windows.
-# see: http://docs.python.org/library/multiprocessing.html#multiprocessing-programming
-#
-if __name__ == '__main__':
- # active run
- cmdline.run (options)
- active_text = open("test_active_if/summary.5").read()
- if active_text != expected_active_text:
- raise Exception("Error:\n\tExpected\n%s\nInstead\n%s\n" % (active_text, expected_active_text))
- os.system("rm -rf test_active_if")
-
-
- # inactive run
- pipeline_active_if = False
- cmdline.run (options)
- inactive_text = open("test_active_if/summary.5").read()
- if inactive_text != expected_inactive_text:
- raise Exception("Error:\n\tExpected\n%s\nInstead\n%s\n" % (inactive_text, expected_inactive_text))
- os.system("rm -rf test_active_if")
+
+
+
+# alternative syntax
+test_pipeline = Pipeline("test")
+test_pipeline.originate(task1, ['test_active_if/a.1', 'test_active_if/b.1'], "an extra_parameter")\
+ .follows(mkdir("test_active_if"))
+test_pipeline.transform(task2, task1, suffix(".1"), ".2")
+test_pipeline.transform(task3, task1, suffix(".1"), ".3").active_if(lambda:pipeline_active_if)
+test_pipeline.collate(task4, [task2, task3], regex(r"(.+)\.[23]"), r"\1.4")
+test_pipeline.merge(task5, task4, "test_active_if/summary.5")
+
+class Test_ruffus(unittest.TestCase):
+ def setUp(self):
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+ os.makedirs(tempdir)
+
+
+
+ def tearDown(self):
+ try:
+ shutil.rmtree(tempdir)
+ pass
+ except:
+ pass
+
+ def test_active_if_true (self):
+ global pipeline_active_if
+ pipeline_active_if = True
+ pipeline_run(multiprocess = 50, verbose = 0)
+
+ with open("test_active_if/summary.5") as ii:
+ active_text = ii.read()
+ if active_text != expected_active_text:
+ raise Exception("Error:\n\tExpected\n%s\nInstead\n%s\n" % (active_text, expected_active_text))
+
+ def test_active_if_false (self):
+ global pipeline_active_if
+ pipeline_active_if = False
+ pipeline_run(multiprocess = 50, verbose = 0)
+ with open("test_active_if/summary.5") as ii:
+ inactive_text = ii.read()
+ if inactive_text != expected_inactive_text:
+ raise Exception("Error:\n\tExpected\n%s\nInstead\n%s\n" % (inactive_text, expected_inactive_text))
+ shutil.rmtree("test_active_if")
+
+ def test_newstyle_active_if_true (self):
+ global pipeline_active_if
+ pipeline_active_if = True
+ test_pipeline.run(multiprocess = 50, verbose = 0)
+
+ with open("test_active_if/summary.5") as ii:
+ active_text = ii.read()
+ if active_text != expected_active_text:
+ raise Exception("Error:\n\tExpected\n%s\nInstead\n%s\n" % (active_text, expected_active_text))
+
+ def test_newstyle_active_if_false (self):
+ global pipeline_active_if
+ pipeline_active_if = False
+ test_pipeline.run(multiprocess = 50, verbose = 0)
+ with open("test_active_if/summary.5") as ii:
+ inactive_text = ii.read()
+ if inactive_text != expected_inactive_text:
+ raise Exception("Error:\n\tExpected\n%s\nInstead\n%s\n" % (inactive_text, expected_inactive_text))
+ shutil.rmtree("test_active_if")
+
+
+
+
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/ruffus/test/test_branching_dependencies.py b/ruffus/test/test_branching_dependencies.py
index 44a2bcd..0d96f70 100755
--- a/ruffus/test/test_branching_dependencies.py
+++ b/ruffus/test/test_branching_dependencies.py
@@ -9,93 +9,29 @@ from __future__ import print_function
"""
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# options
+import os
+import sys
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-from optparse import OptionParser
-import sys, os
-import os.path
-try:
- import StringIO as io
-except:
- import io as io
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
-import re
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-if __name__ == '__main__':
- module_name = os.path.split(sys.argv[0])[1]
- module_name = os.path.splitext(module_name)[0];
-else:
- module_name = __name__
-
-
-
-
-parser = OptionParser(version="%prog 1.0")
-parser.add_option("-D", "--debug", dest="debug",
- action="store_true", default=False,
- help="Make sure output is correct and clean up.")
-parser.add_option("-t", "--target_tasks", dest="target_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Target task(s) of pipeline.")
-parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Pipeline task(s) which will be included even if they are up to date.")
-parser.add_option("-j", "--jobs", dest="jobs",
- default=1,
- metavar="jobs",
- type="int",
- help="Specifies the number of jobs (commands) to run simultaneously.")
-parser.add_option("-v", "--verbose", dest = "verbose",
- action="count", default=0,
- help="Do not echo to shell but only print to log.")
-parser.add_option("--touch_files_only", dest = "touch_files_only",
- action="store_true", default=False,
- help="Do not run pipeline. Only touch.")
-parser.add_option("-d", "--dependency", dest="dependency_file",
- #default="simple.svg",
- metavar="FILE",
- type="string",
- help="Print a dependency graph of the pipeline that would be executed "
- "to FILE, but do not execute it.")
-parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
- metavar="FORMAT",
- type="string",
- default = 'svg',
- help="format of dependency graph file. Can be 'ps' (PostScript), "+
- "'svg' 'svgz' (Structured Vector Graphics), " +
- "'png' 'gif' (bitmap graphics) etc ")
-parser.add_option("-n", "--just_print", dest="just_print",
- action="store_true", default=False,
- help="Print a description of the jobs that would be executed, "
- "but do not execute them.")
-parser.add_option("-M", "--minimal_rebuild_mode", dest="minimal_rebuild_mode",
- action="store_true", default=False,
- help="Rebuild a minimum of tasks necessary for the target. "
- "Ignore upstream out of date tasks if intervening tasks are fine.")
-parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
- action="store_true", default=False,
- help="Do not print out legend and key for dependency graph.")
-parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
- action="store_true", default=False,
- help="Draw horizontal dependency graph.")
-
-parameters = [
- ]
+try:
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
@@ -109,23 +45,24 @@ parameters = [
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+import unittest
+import shutil
import time
import re
-import operator
-import sys,os
from collections import defaultdict
-import random
-sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-import ruffus
+try:
+ from StringIO import StringIO
+except:
+ from io import StringIO
# use simplejson in place of json for python < 2.6
-try:
- import json
-except ImportError:
- import simplejson
- json = simplejson
+import json
+#try:
+# import json
+#except ImportError:
+# import simplejson
+# json = simplejson
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -144,17 +81,19 @@ def test_job_io(infiles, outfiles, extra_params):
if isinstance(infiles, str):
infiles = [infiles]
- elif infiles == None:
+ elif infiles is None:
infiles = []
if isinstance(outfiles, str):
outfiles = [outfiles]
output_text = list()
for f in infiles:
- output_text.append(open(f).read())
+ with open(f) as ii:
+ output_text.append(ii.read())
output_text = "".join(sorted(output_text))
output_text += json.dumps(infiles) + " -> " + json.dumps(outfiles) + "\n"
for f in outfiles:
- open(f, "w").write(output_text)
+ with open(f, "w") as oo:
+ oo.write(output_text)
@@ -169,11 +108,6 @@ def test_job_io(infiles, outfiles, extra_params):
-# get help string
-f =io.StringIO()
-parser.print_help(f)
-helpstr = f.getvalue()
-(options, remaining_args) = parser.parse_args()
@@ -190,35 +124,43 @@ helpstr = f.getvalue()
# -> 4 ->
# 5 -> 6
#
-
tempdir = "temp_branching_dir/"
+
+def do_write(file_name, what):
+ with open(file_name, "a") as oo:
+ oo.write(what)
+test_file = tempdir + "task.done"
#
# task1
#
@originate([tempdir + d for d in ('a.1', 'b.1', 'c.1')])
@follows(mkdir(tempdir))
- at posttask(lambda: open(tempdir + "task.done", "a").write("Task 1 Done\n"))
+ at posttask(lambda: do_write(test_file, "Task 1 Done\n"))
def task1(outfile, *extra_params):
"""
First task
"""
- open(tempdir + "jobs.start", "a").write('job = %s\n' % json.dumps([None, outfile]))
+ with open(tempdir + "jobs.start", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([None, outfile]))
test_job_io(None, outfile, extra_params)
- open(tempdir + "jobs.finish", "a").write('job = %s\n' % json.dumps([None, outfile]))
+ with open(tempdir + "jobs.finish", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([None, outfile]))
#
# task2
#
- at posttask(lambda: open(tempdir + "task.done", "a").write("Task 2 Done\n"))
+ at posttask(lambda: do_write(test_file, "Task 2 Done\n"))
@transform(task1, suffix(".1"), ".2")
def task2(infiles, outfiles, *extra_params):
"""
Second task
"""
- open(tempdir + "jobs.start", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.start", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
test_job_io(infiles, outfiles, extra_params)
- open(tempdir + "jobs.finish", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.finish", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
@@ -226,14 +168,16 @@ def task2(infiles, outfiles, *extra_params):
# task3
#
@transform(task2, regex('(.*).2'), inputs([r"\1.2", tempdir + "a.1"]), r'\1.3')
- at posttask(lambda: open(tempdir + "task.done", "a").write("Task 3 Done\n"))
+ at posttask(lambda: do_write(test_file, "Task 3 Done\n"))
def task3(infiles, outfiles, *extra_params):
"""
Third task
"""
- open(tempdir + "jobs.start", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.start", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
test_job_io(infiles, outfiles, extra_params)
- open(tempdir + "jobs.finish", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.finish", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
@@ -243,30 +187,34 @@ def task3(infiles, outfiles, *extra_params):
@jobs_limit(1)
@transform(tempdir + "*.1", suffix(".1"), ".4")
@follows(task1)
- at posttask(lambda: open(tempdir + "task.done", "a").write("Task 4 Done\n"))
+ at posttask(lambda: do_write(test_file, "Task 4 Done\n"))
def task4(infiles, outfiles, *extra_params):
"""
Fourth task is extra slow
"""
- open(tempdir + "jobs.start", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.start", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
time.sleep(0.1)
test_job_io(infiles, outfiles, extra_params)
- open(tempdir + "jobs.finish", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.finish", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
#
# task5
#
@files(None, tempdir + 'a.5')
@follows(mkdir(tempdir))
- at posttask(lambda: open(tempdir + "task.done", "a").write("Task 5 Done\n"))
+ at posttask(lambda: do_write(test_file, "Task 5 Done\n"))
def task5(infiles, outfiles, *extra_params):
"""
Fifth task is extra slow
"""
- open(tempdir + "jobs.start", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.start", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
time.sleep(1)
test_job_io(infiles, outfiles, extra_params)
- open(tempdir + "jobs.finish", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.finish", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
#
# task6
@@ -274,17 +222,55 @@ def task5(infiles, outfiles, *extra_params):
#@files([[[tempdir + d for d in 'a.3', 'b.3', 'c.3', 'a.4', 'b.4', 'c.4', 'a.5'], tempdir + 'final.6']])
@merge([task3, task4, task5], tempdir + "final.6")
@follows(task3, task4, task5, )
- at posttask(lambda: open(tempdir + "task.done", "a").write("Task 6 Done\n"))
+ at posttask(lambda: do_write(test_file, "Task 6 Done\n"))
def task6(infiles, outfiles, *extra_params):
"""
final task
"""
- open(tempdir + "jobs.start", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.start", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
test_job_io(infiles, outfiles, extra_params)
- open(tempdir + "jobs.finish", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.finish", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
+
+
+
+
+
+#
+# Use equivalent but new sytle syntax
+#
+test_pipeline = Pipeline("test")
+
+test_pipeline.originate(task_func = task1,
+ output = [tempdir + d for d in ('a.1', 'b.1', 'c.1')])\
+ .follows(mkdir(tempdir))\
+ .posttask(lambda: do_write(test_file, "Task 1 Done\n"))
+
+test_pipeline.transform(task_func = task2,
+ input = task1,
+ filter = suffix(".1"),
+ output = ".2") \
+ .posttask(lambda: do_write(test_file, "Task 2 Done\n"))
+
+test_pipeline.transform(task3, task2, regex('(.*).2'), inputs([r"\1.2", tempdir + "a.1"]), r'\1.3')\
+ .posttask(lambda: do_write(test_file, "Task 3 Done\n"))
+
+test_pipeline.transform(task4, tempdir + "*.1", suffix(".1"), ".4")\
+ .follows(task1)\
+ .posttask(lambda: do_write(test_file, "Task 4 Done\n"))\
+ .jobs_limit(1)
+test_pipeline.files(task5, None, tempdir + 'a.5')\
+ .follows(mkdir(tempdir))\
+ .posttask(lambda: do_write(test_file, "Task 5 Done\n"))
+test_pipeline.merge(task_func = task6,
+ input = [task3, task4, task5],
+ output = tempdir + "final.6")\
+ .follows(task3, task4, task5, ) \
+ .posttask(lambda: do_write(test_file, "Task 6 Done\n"))
def check_job_order_correct(filename):
@@ -303,11 +289,12 @@ def check_job_order_correct(filename):
index_re = re.compile(r'.*\.([0-9])["\]\n]*$')
job_indices = defaultdict(list)
- for linenum, l in enumerate(open(filename)):
- m = index_re.search(l)
- if not m:
- raise "Non-matching line in [%s]" % filename
- job_indices[int(m.group(1))].append(linenum)
+ with open(filename) as ii:
+ for linenum, l in enumerate(ii):
+ m = index_re.search(l)
+ if not m:
+ raise "Non-matching line in [%s]" % filename
+ job_indices[int(m.group(1))].append(linenum)
for job_index in job_indices:
job_indices[job_index].sort()
@@ -354,7 +341,8 @@ def check_final_output_correct(after_touch_files = False):
orig_expected_output = expected_output
if after_touch_files:
expected_output.pop(-3)
- final_6_contents = sorted([l.rstrip() for l in open(tempdir + "final.6", "r").readlines()])
+ with open(tempdir + "final.6", "r") as ii:
+ final_6_contents = sorted([l.rstrip() for l in ii.readlines()])
if final_6_contents != expected_output:
print("Actual:", file=sys.stderr)
for ll in final_6_contents:
@@ -370,83 +358,89 @@ def check_final_output_correct(after_touch_files = False):
raise Exception ("Final.6 output is not as expected\n")
-#
-# Necessary to protect the "entry point" of the program under windows.
-# see: http://docs.python.org/library/multiprocessing.html#multiprocessing-programming
-#
-if __name__ == '__main__':
- print("Python version %s" % sys.version, file=sys.stderr)
- print("Ruffus version %s" % ruffus.__version__, file=sys.stderr)
- if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
- verbose=options.verbose)
-
- elif options.dependency_file:
- pipeline_printout_graph ( open(options.dependency_file, "w"),
- options.dependency_graph_format,
- options.target_tasks,
- options.forced_tasks,
- draw_vertically = not options.draw_horizontally,
- no_key_legend = options.no_key_legend_in_graph)
-
- elif options.debug:
- import os
- os.system("rm -rf %s" % tempdir)
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- logger = stderr_logger if options.verbose else black_hole_logger,
- verbose = options.verbose)
+class Test_ruffus(unittest.TestCase):
+
+ def tearDown(self):
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+ def setUp(self):
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+ os.makedirs(tempdir)
+
+ def test_ruffus (self):
+ print("\n\n Run pipeline normally...")
+ pipeline_run(multiprocess = 10, verbose=0)
+ check_final_output_correct()
+ check_job_order_correct(tempdir + "jobs.start")
+ check_job_order_correct(tempdir + "jobs.finish")
+ print(" OK")
+
+ print("\n\n Touch task2 only:")
+ os.unlink(os.path.join(tempdir, "jobs.start") )
+ os.unlink(os.path.join(tempdir, "jobs.finish") )
+ print(" First delete b.1 for task2...")
+ os.unlink(os.path.join(tempdir, "b.1"))
+ print(" Then run with touch_file_only...")
+ pipeline_run([task2], multiprocess = 10, touch_files_only=True, verbose = 0)
+
+ # check touching has made task2 up to date
+ s = StringIO()
+ pipeline_printout(s, [task2], verbose=4, wrap_width = 10000)
+ output_str = s.getvalue()
+ #print (">>>\n", output_str, "<<<\n", file=sys.stderr)
+ if "b.1" in output_str:
+ raise Exception("Expected b.1 created by touching...")
+ if "b.2" in output_str:
+ raise Exception("Expected b.2 created by touching...")
+ print(" Touching has made task2 up to date...\n")
+
+ print(" Then run normally again...")
+ pipeline_run(multiprocess = 10, verbose=0)
+ check_final_output_correct(True)
+ check_job_order_correct(tempdir + "jobs.start")
+ check_job_order_correct(tempdir + "jobs.finish")
+
+ def test_ruffus_new_syntax (self):
+ print("\n\n Run pipeline normally...")
+ test_pipeline.run(multiprocess = 10, verbose=0)
check_final_output_correct()
check_job_order_correct(tempdir + "jobs.start")
check_job_order_correct(tempdir + "jobs.finish")
+ print(" OK")
+
+ print("\n\n Touch task2 only:")
+ os.unlink(os.path.join(tempdir, "jobs.start") )
+ os.unlink(os.path.join(tempdir, "jobs.finish") )
+ print(" First delete b.1 for task2...")
+ os.unlink(os.path.join(tempdir, "b.1"))
+ print(" Then run with touch_file_only...")
+ test_pipeline.run([task2], multiprocess = 10, touch_files_only=True, verbose = 0)
+
+ # check touching has made task2 up to date
+ s = StringIO()
+ test_pipeline.printout(s, [task2], verbose=4, wrap_width = 10000)
+ output_str = s.getvalue()
+ #print (">>>\n", output_str, "<<<\n", file=sys.stderr)
+ if "b.1" in output_str:
+ raise Exception("Expected b.1 created by touching...")
+ if "b.2" in output_str:
+ raise Exception("Expected b.2 created by touching...")
+ print(" Touching has made task2 up to date...\n")
+
+ print(" Then run normally again...")
+ test_pipeline.run(multiprocess = 10, verbose=0)
+ check_final_output_correct(True)
+ check_job_order_correct(tempdir + "jobs.start")
+ check_job_order_correct(tempdir + "jobs.finish")
+
+if __name__ == '__main__':
+ unittest.main()
- #
- # check touch file works, running the pipeline leaving an empty file where b.1
- # would be
- #
- if options.touch_files_only:
- #
- # remove these because the precedence for the two runs must not be mixed together
- #
- os.unlink(os.path.join(tempdir, "jobs.start") )
- os.unlink(os.path.join(tempdir, "jobs.finish") )
-
- #
- # remove b.1 and touch
- #
- if options.verbose:
- print("\n\nNow just delete b.1 for task2...\n")
- os.unlink(os.path.join(tempdir, "b.1"))
- pipeline_run([task2], options.forced_tasks, multiprocess = options.jobs,
- logger = stderr_logger if options.verbose else black_hole_logger,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- verbose = options.verbose,
- touch_files_only = options.touch_files_only)
-
-
- #
- # Now wait for the empty b.1 to show up in the output
- #
- if options.verbose:
- print("\n\nRun normally...\n")
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- logger = stderr_logger if options.verbose else black_hole_logger,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- verbose = options.verbose)
- check_final_output_correct(options.touch_files_only)
- check_job_order_correct(tempdir + "jobs.start")
- check_job_order_correct(tempdir + "jobs.finish")
-
-
-
- print("OK")
- import shutil
- shutil.rmtree(tempdir)
- else:
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- logger = stderr_logger if options.verbose else black_hole_logger,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- verbose = options.verbose, touch_files_only = options.touch_files_only)
- print("OK")
diff --git a/ruffus/test/test_cmdline.py b/ruffus/test/test_cmdline.py
index 67d35ea..9573ae7 100755
--- a/ruffus/test/test_cmdline.py
+++ b/ruffus/test/test_cmdline.py
@@ -8,22 +8,41 @@ from __future__ import print_function
"""
+import os
+import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+
+try:
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
+
+handle_verbose = ruffus.cmdline.handle_verbose
+cmdline= ruffus.cmdline
import unittest
-import os, re
-import sys
+import re
import shutil
-try:
- from StringIO import StringIO
-except:
- from io import StringIO
-import time
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0, os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus.cmdline import handle_verbose
-import ruffus.cmdline as cmdline
+
+
+
# mock for command line options
class t_options(object):
@@ -47,7 +66,7 @@ class Test_cmdline(unittest.TestCase):
# s = StringIO()
# cleanup_tmpdir()
# pipeline_printout(s, [test_regex_task], verbose=5, wrap_width = 10000)
- # self.assertTrue(re.search('Missing files\n\s+\[tmp_test_regex_error_messages/a_name.tmp1, tmp_test_regex_error_messages/a_name.tmp2', s.getvalue()))
+ # self.assertTrue(re.search('Missing files.*\[tmp_test_regex_error_messages/a_name.tmp1, tmp_test_regex_error_messages/a_name.tmp2', s.getvalue(), re.DOTALL))
# self.assertIn("Warning: File match failure: File 'tmp_test_regex_error_messages/a_name.tmp1' does not match regex", s.getvalue())
# self.assertRaisesRegex(fatal_error_input_file_does_not_match,
# "File '.*?' does not match regex\('.*?'\) and pattern '.*?':\n.*unknown group name",
@@ -81,14 +100,14 @@ class Test_cmdline(unittest.TestCase):
setattr(options, "verbose", None)
handle_verbose(options)
self.assertTrue(options.verbose==None)
- self.assertTrue(options.verbose_abbreviated_path == None)
+ self.assertTrue(options.verbose_abbreviated_path is None)
# options.verbose defined by user to be 0
options = t_options()
setattr(options, "verbose", 0)
handle_verbose(options)
self.assertTrue(options.verbose==0)
- self.assertTrue(options.verbose_abbreviated_path == None)
+ self.assertTrue(options.verbose_abbreviated_path is None)
# options.verbose defined by user to be "6"
options = t_options()
diff --git a/ruffus/test/test_collate.py b/ruffus/test/test_collate.py
index b7bd445..851f188 100755
--- a/ruffus/test/test_collate.py
+++ b/ruffus/test/test_collate.py
@@ -8,90 +8,29 @@ from __future__ import print_function
"""
+import os
+import sys
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
-# options
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
-from optparse import OptionParser
-import sys, os
-import os.path
try:
- import StringIO as io
-except:
- import io as io
-import re
-
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-if __name__ == '__main__':
- module_name = os.path.split(sys.argv[0])[1]
- module_name = os.path.splitext(module_name)[0];
-else:
- module_name = __name__
-
-
-
-
-parser = OptionParser(version="%prog 1.0")
-parser.add_option("-D", "--debug", dest="debug",
- action="store_true", default=False,
- help="Make sure output is correct and clean up.")
-parser.add_option("-t", "--target_tasks", dest="target_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Target task(s) of pipeline.")
-parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Pipeline task(s) which will be included even if they are up to date.")
-parser.add_option("-j", "--jobs", dest="jobs",
- default=1,
- metavar="jobs",
- type="int",
- help="Specifies the number of jobs (commands) to run simultaneously.")
-parser.add_option("-v", "--verbose", dest = "verbose",
- action="count", default=0,
- help="Do not echo to shell but only print to log.")
-parser.add_option("-d", "--dependency", dest="dependency_file",
- #default="simple.svg",
- metavar="FILE",
- type="string",
- help="Print a dependency graph of the pipeline that would be executed "
- "to FILE, but do not execute it.")
-parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
- metavar="FORMAT",
- type="string",
- default = 'svg',
- help="format of dependency graph file. Can be 'ps' (PostScript), "+
- "'svg' 'svgz' (Structured Vector Graphics), " +
- "'png' 'gif' (bitmap graphics) etc ")
-parser.add_option("-n", "--just_print", dest="just_print",
- action="store_true", default=False,
- help="Print a description of the jobs that would be executed, "
- "but do not execute them.")
-parser.add_option("-M", "--minimal_rebuild_mode", dest="minimal_rebuild_mode",
- action="store_true", default=False,
- help="Rebuild a minimum of tasks necessary for the target. "
- "Ignore upstream out of date tasks if intervening tasks are fine.")
-parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
- action="store_true", default=False,
- help="Do not print out legend and key for dependency graph.")
-parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
- action="store_true", default=False,
- help="Draw horizontal dependency graph.")
-
-parameters = [
- ]
-
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
@@ -105,21 +44,21 @@ parameters = [
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-import re
-import operator
-import sys,os
+import unittest
+import shutil
+try:
+ from StringIO import StringIO
+except:
+ from io import StringIO
from collections import defaultdict
-import random
-
-sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
# use simplejson in place of json for python < 2.6
-try:
- import json
-except ImportError:
- import simplejson
- json = simplejson
+import json
+#try:
+# import json
+#except ImportError:
+# import simplejson
+# json = simplejson
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -132,11 +71,6 @@ except ImportError:
-# get help string
-f =io.StringIO()
-parser.print_help(f)
-helpstr = f.getvalue()
-(options, remaining_args) = parser.parse_args()
species_list = defaultdict(list)
species_list["mammals"].append("cow" )
@@ -160,8 +94,15 @@ tempdir = "temp_filesre_combine/"
#
# task1
#
+
+def do_write(file_name, what):
+ with open(file_name, "a") as oo:
+ oo.write(what)
+test_file = tempdir + "task.done"
+
+
@follows(mkdir(tempdir, tempdir + "test"))
- at posttask(lambda: open(tempdir + "task.done", "a").write("Task 1 Done\n"))
+ at posttask(lambda: do_write(tempdir + "task.done", "Task 1 Done\n"))
@split(None, tempdir + '*.animal')
def prepare_files (no_inputs, outputs):
# cleanup previous
@@ -171,23 +112,27 @@ def prepare_files (no_inputs, outputs):
for grouping in species_list:
for species_name in species_list[grouping]:
filename = tempdir + "%s.%s.animal" % (species_name, grouping)
- open(filename, "w").write(species_name + "\n")
+ with open(filename, "w") as oo:
+ oo.write(species_name + "\n")
#
# task2
#
@collate(prepare_files, regex(r'(.*/).*\.(.*)\.animal'), r'\1\2.results')
- at posttask(lambda: open(tempdir + "task.done", "a").write("Task 2 Done\n"))
+ at posttask(lambda: do_write(tempdir + "task.done", "Task 2 Done\n"))
def summarise_by_grouping(infiles, outfile):
"""
Summarise by each species group, e.g. mammals, reptiles, fish
"""
- open(tempdir + "jobs.start", "a").write('job = %s\n' % json.dumps([infiles, outfile]))
- o = open(outfile, "w")
- for i in infiles:
- o.write(open(i).read())
- open(tempdir + "jobs.finish", "a").write('job = %s\n' % json.dumps([infiles, outfile]))
+ with open(tempdir + "jobs.start", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfile]))
+ with open(outfile, "w") as oo:
+ for i in infiles:
+ with open(i) as ii:
+ oo.write(ii.read())
+ with open(tempdir + "jobs.finish", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfile]))
@@ -211,47 +156,51 @@ def check_species_correct():
# -> fish.results
"""
for grouping in species_list:
- assert(open(tempdir + grouping + ".results").read() ==
- "".join(s + "\n" for s in sorted(species_list[grouping])))
+ with open(tempdir + grouping + ".results") as oo:
+ assert(oo.read() == "".join(s + "\n" for s in sorted(species_list[grouping])))
-#
-# Necessary to protect the "entry point" of the program under windows.
-# see: http://docs.python.org/library/multiprocessing.html#multiprocessing-programming
-#
-if __name__ == '__main__':
- if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
- long_winded=True,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
-
- elif options.dependency_file:
- pipeline_printout_graph ( open(options.dependency_file, "w"),
- options.dependency_graph_format,
- options.target_tasks,
- options.forced_tasks,
- draw_vertically = not options.draw_horizontally,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- no_key_legend = options.no_key_legend_in_graph)
- elif options.debug:
- import os
- os.system("rm -rf %s" % tempdir)
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- logger = stderr_logger if options.verbose else black_hole_logger,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- verbose = options.verbose > 1)
+class Test_ruffus(unittest.TestCase):
+ def setUp(self):
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+
+ def tearDown(self):
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+
+ def test_ruffus (self):
+ pipeline_run(multiprocess = 10, verbose = 0)
check_species_correct()
- os.system("rm -rf %s" % tempdir)
- print("OK")
- else:
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- logger = stderr_logger if options.verbose else black_hole_logger,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- verbose = options.verbose > 1)
+ def test_newstyle_ruffus (self):
+ test_pipeline = Pipeline("test")
+ test_pipeline.split(task_func = prepare_files,
+ input = None,
+ output = tempdir + '*.animal')\
+ .follows(mkdir(tempdir, tempdir + "test"))\
+ .posttask(lambda: do_write(tempdir + "task.done", "Task 1 Done\n"))
+
+ test_pipeline.collate(task_func = summarise_by_grouping,
+ input = prepare_files,
+ filter = regex(r'(.*/).*\.(.*)\.animal'),
+ output = r'\1\2.results')\
+ .posttask(lambda: do_write(tempdir + "task.done", "Task 2 Done\n"))
+
+ test_pipeline.run(multiprocess = 10, verbose = 0)
+ check_species_correct()
+
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/ruffus/test/test_combinatorics.py b/ruffus/test/test_combinatorics.py
index a4b5d04..a975b2e 100755
--- a/ruffus/test/test_combinatorics.py
+++ b/ruffus/test/test_combinatorics.py
@@ -8,27 +8,45 @@ from __future__ import print_function
"""
-
-import unittest
import os
import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+try:
+ attrlist = ruffus.combinatorics.__all__
+except AttributeError:
+ attrlist = dir (ruffus.combinatorics)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus.combinatorics, attr)
+
+for attr in "pipeline_run", "pipeline_printout", "suffix", "transform", "split", "merge", "dbdict", "follows", "Pipeline", "formatter", "output_from", "originate":
+ globals()[attr] = getattr (ruffus, attr)
+RethrownJobError = ruffus.ruffus_exceptions.RethrownJobError
+RUFFUS_HISTORY_FILE = ruffus.ruffus_utility.RUFFUS_HISTORY_FILE
+CHECKSUM_FILE_TIMESTAMPS = ruffus.ruffus_utility.CHECKSUM_FILE_TIMESTAMPS
+
+
+
+import unittest
import shutil
try:
from StringIO import StringIO
except:
from io import StringIO
-import time
import re
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0, os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-from ruffus import (pipeline_run, pipeline_printout, suffix, transform, split,
- merge, dbdict, follows)
-from ruffus.combinatorics import *
-from ruffus.ruffus_exceptions import RethrownJobError
-from ruffus.ruffus_utility import (RUFFUS_HISTORY_FILE,
- CHECKSUM_FILE_TIMESTAMPS)
workdir = 'tmp_test_combinatorics'
#sub-1s resolution in system?
@@ -44,7 +62,7 @@ def touch (filename):
#
# generate_initial_files1
#___________________________________________________________________________
- at originate([workdir + "/" + prefix + "_name.tmp1" for prefix in "abcd"])
+ at originate(output = [workdir + "/" + prefix + "_name.tmp1" for prefix in "abcd"])
def generate_initial_files1(out_name):
with open(out_name, 'w') as outfile:
pass
@@ -77,12 +95,12 @@ def generate_initial_files3(out_name):
formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
generate_initial_files2,
formatter(),
- generate_initial_files3,
- formatter(r"tmp1$" ),
"{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.{basename[2][0]}.tmp2",
- "{basename[0][0][0]}{basename[1][0][0]}{basename[2][0][0]}", # extra: prefices only (abcd etc)
- "{subpath[0][0][0]}", # extra: path for 2nd input, 1st file
- "{subdir[0][0][0]}")
+ input3 = generate_initial_files3,
+ filter3 = formatter(r"tmp1$" ),
+ extras = [ "{basename[0][0][0]}{basename[1][0][0]}{basename[2][0][0]}", # extra: prefices only (abcd etc)
+ "{subpath[0][0][0]}", # extra: path for 2nd input, 1st file
+ "{subdir[0][0][0]}"])
def test_product_task( infiles, outfile,
prefices,
subpath,
@@ -273,13 +291,13 @@ def test_permutations3_merged_task( infiles, outfile):
# test_combinations_with_replacement2_task
#___________________________________________________________________________
@combinations_with_replacement(
- generate_initial_files1,
- formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
- 2,
- "{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.tmp2",
- "{basename[0][0][0]}{basename[1][0][0]}", # extra: prefices
+ input = generate_initial_files1,
+ filter = formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
+ tuple_size = 2,
+ output = "{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.tmp2",
+ extras = ["{basename[0][0][0]}{basename[1][0][0]}", # extra: prefices
"{subpath[0][0][0]}", # extra: path for 2nd input, 1st file
- "{subdir[0][0][0]}")
+ "{subdir[0][0][0]}"])
def test_combinations_with_replacement2_task( infiles, outfile,
prefices,
subpath,
@@ -349,11 +367,11 @@ class TestCombinatorics(unittest.TestCase):
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [test_product_merged_task], verbose=5, wrap_width = 10000)
- self.assertTrue(re.search('Job needs update: Missing files\n\s+'
+ self.assertTrue(re.search('Job needs update:.*Missing files.*'
'\[.*tmp_test_combinatorics/a_name.tmp1, '
'.*tmp_test_combinatorics/e_name.tmp1, '
'.*tmp_test_combinatorics/h_name.tmp1, '
- '.*tmp_test_combinatorics/a_name.e_name.h_name.tmp2\]', s.getvalue()))
+ '.*tmp_test_combinatorics/a_name.e_name.h_name.tmp2\]', s.getvalue(), re.DOTALL))
def test_product_run(self):
"""Run product"""
@@ -415,10 +433,10 @@ class TestCombinatorics(unittest.TestCase):
s = StringIO()
pipeline_printout(s, [test_combinations2_merged_task], verbose=5, wrap_width = 10000)
- self.assertTrue(re.search('Job needs update: Missing files\n\s+'
+ self.assertTrue(re.search('Job needs update:.*Missing files.*'
'\[.*tmp_test_combinatorics/a_name.tmp1, '
'.*tmp_test_combinatorics/b_name.tmp1, '
- '.*tmp_test_combinatorics/a_name.b_name.tmp2\]', s.getvalue()))
+ '.*tmp_test_combinatorics/a_name.b_name.tmp2\]', s.getvalue(), re.DOTALL))
def test_combinations2_run(self):
@@ -557,7 +575,6 @@ class TestCombinatorics(unittest.TestCase):
# cleanup
#___________________________________________________________________________
def tearDown(self):
- pass
shutil.rmtree(workdir)
diff --git a/ruffus/test/test_empty_files_decorator.py b/ruffus/test/test_empty_files_decorator.py
index 900e0c6..a4e8788 100755
--- a/ruffus/test/test_empty_files_decorator.py
+++ b/ruffus/test/test_empty_files_decorator.py
@@ -3,95 +3,34 @@ from __future__ import print_function
"""
test_pausing.py
-
+
test time.sleep keeping input files and output file times correct
"""
+import os
+import sys
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
-# options
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
-from optparse import OptionParser
-import sys, os
-import os.path
try:
- import StringIO as io
-except:
- import io as io
-import re
-
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-if __name__ == '__main__':
- module_name = os.path.split(sys.argv[0])[1]
- module_name = os.path.splitext(module_name)[0];
-else:
- module_name = __name__
-
-
-
-
-parser = OptionParser(version="%prog 1.0")
-parser.add_option("-D", "--debug", dest="debug",
- action="store_true", default=False,
- help="Make sure output is correct and clean up.")
-parser.add_option("-t", "--target_tasks", dest="target_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Target task(s) of pipeline.")
-parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Pipeline task(s) which will be included even if they are up to date.")
-parser.add_option("-j", "--jobs", dest="jobs",
- default=1,
- metavar="jobs",
- type="int",
- help="Specifies the number of jobs (commands) to run simultaneously.")
-parser.add_option("-v", "--verbose", dest = "verbose",
- action="count", default=0,
- help="Do not echo to shell but only print to log.")
-parser.add_option("-d", "--dependency", dest="dependency_file",
- #default="simple.svg",
- metavar="FILE",
- type="string",
- help="Print a dependency graph of the pipeline that would be executed "
- "to FILE, but do not execute it.")
-parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
- metavar="FORMAT",
- type="string",
- default = 'svg',
- help="format of dependency graph file. Can be 'ps' (PostScript), "+
- "'svg' 'svgz' (Structured Vector Graphics), " +
- "'png' 'gif' (bitmap graphics) etc ")
-parser.add_option("-n", "--just_print", dest="just_print",
- action="store_true", default=False,
- help="Print a description of the jobs that would be executed, "
- "but do not execute them.")
-parser.add_option("-M", "--minimal_rebuild_mode", dest="minimal_rebuild_mode",
- action="store_true", default=False,
- help="Rebuild a minimum of tasks necessary for the target. "
- "Ignore upstream out of date tasks if intervening tasks are fine.")
-parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
- action="store_true", default=False,
- help="Do not print out legend and key for dependency graph.")
-parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
- action="store_true", default=False,
- help="Draw horizontal dependency graph.")
-
-parameters = [
- ]
-
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
@@ -100,20 +39,20 @@ parameters = [
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-# imports
+# imports
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
import re
import operator
-import sys,os
+try:
+ import StringIO as io
+except:
+ import io as io
from collections import defaultdict
import random
-sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-
# use simplejson in place of json for python < 2.6
try:
import json
@@ -121,7 +60,7 @@ except ImportError:
import simplejson
json = simplejson
-
+
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -134,15 +73,6 @@ except ImportError:
-# get help string
-f =io.StringIO()
-parser.print_help(f)
-helpstr = f.getvalue()
-(options, remaining_args) = parser.parse_args()
-
-
-
-
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -162,53 +92,51 @@ def task1():
"""
print("Task1", file=sys.stderr)
-
+
import unittest
-class Test_task(unittest.TestCase):
+class t_save_to_str_logger:
+ """
+ Everything to stderr
+ """
+ def __init__ (self):
+ self.info_str = ""
+ self.warning_str = ""
+ self.debug_str = ""
+ def info (self, message):
+ self.info_str += message
+ def warning (self, message):
+ self.warning_str += message
+ def debug (self, message):
+ self.debug_str += message
+class Test_task(unittest.TestCase):
def test_task (self):
- class t_save_to_str_logger:
- """
- Everything to stderr
- """
- def __init__ (self):
- self.info_str = ""
- self.warning_str = ""
- self.debug_str = ""
- def info (self, message):
- self.info_str += message
- def warning (self, message):
- self.warning_str += message
- def debug (self, message):
- self.debug_str += message
save_to_str_logger = t_save_to_str_logger()
- pipeline_run([task1], options.forced_tasks, multiprocess = options.jobs,
+ pipeline_run(multiprocess = 10,
logger = save_to_str_logger,
verbose = 1)
self.assertTrue("@files() was empty" in save_to_str_logger.warning_str)
print("\n Warning printed out correctly", file=sys.stderr)
-
-#
+
+
+ def test_newstyle_task (self):
+ test_pipeline = Pipeline("test")
+ test_pipeline.files(task1, a)
+
+ save_to_str_logger = t_save_to_str_logger()
+ test_pipeline.run(multiprocess = 10,
+ logger = save_to_str_logger,
+ verbose = 1)
+ self.assertTrue("@files() was empty" in save_to_str_logger.warning_str)
+ print("\n Warning printed out correctly", file=sys.stderr)
+
+#
# Necessary to protect the "entry point" of the program under windows.
# see: http://docs.python.org/library/multiprocessing.html#multiprocessing-programming
#
if __name__ == '__main__':
- if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
- long_winded=True,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
-
- elif options.dependency_file:
- pipeline_printout_graph ( open(options.dependency_file, "w"),
- options.dependency_graph_format,
- options.target_tasks,
- options.forced_tasks,
- draw_vertically = not options.draw_horizontally,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- no_key_legend = options.no_key_legend_in_graph)
- else:
- unittest.main()
+ unittest.main()
diff --git a/ruffus/test/test_exceptions.py b/ruffus/test/test_exceptions.py
index a995cbf..6a6ff84 100755
--- a/ruffus/test/test_exceptions.py
+++ b/ruffus/test/test_exceptions.py
@@ -4,36 +4,26 @@ from __future__ import print_function
test_exceptions.py
- use :
- --debug to test automatically
- -j N / --jobs N to specify multitasking
- -v to see the jobs in action
- -n / --just_print to see what jobs would run
-
"""
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# options
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-import sys, os
-
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-
+import os
+import sys
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
-from ruffus import *
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
-parser = cmdline.get_argparse( description='Test exceptions?')
-options = parser.parse_args()
-# optional logger which can be passed to ruffus tasks
-logger, logger_mutex = cmdline.setup_logging (__name__, options.log_file, options.verbose)
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+for attr in "parallel", "pipeline_run", "Pipeline":
+ globals()[attr] = getattr (ruffus, attr)
@@ -46,14 +36,39 @@ logger, logger_mutex = cmdline.setup_logging (__name__, options.log_file, option
@parallel([['A', 1], ['B',3], ['C',3], ['D',4], ['E',4], ['F',4]])
def parallel_task(name, param1):
- if options.verbose:
- sys.stderr.write(" Parallel task %s: \n\n" % name)
+ sys.stderr.write(" Parallel task %s: \n" % name)
#raise task.JobSignalledBreak("Oops! I did it again!")
+ print (" Raising exception", file = sys.stderr)
raise Exception("new")
+
+import unittest, shutil
try:
- cmdline.run (options, logger = logger, log_exceptions = True)
+ from StringIO import StringIO
except:
- pass
+ from io import StringIO
+
+class Test_ruffus(unittest.TestCase):
+ def test_ruffus (self):
+ try:
+ pipeline_run(multiprocess = 50, verbose = 0)
+ except ruffus.ruffus_exceptions.RethrownJobError:
+ return
+ raise Exception("Missing exception")
+
+ def test_newstyle_ruffus (self):
+ test_pipeline = Pipeline("test")
+ test_pipeline.parallel(parallel_task, [['A', 1], ['B',3], ['C',3], ['D',4], ['E',4], ['F',4]])
+ try:
+ test_pipeline.run(multiprocess = 50, verbose = 0)
+ except ruffus.ruffus_exceptions.RethrownJobError:
+ return
+ raise Exception("Missing exception")
+
+
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/ruffus/test/test_file_name_parameters.py b/ruffus/test/test_file_name_parameters.py
index b61201b..2e92528 100755
--- a/ruffus/test/test_file_name_parameters.py
+++ b/ruffus/test/test_file_name_parameters.py
@@ -1,30 +1,5 @@
#!/usr/bin/env python
from __future__ import print_function
-################################################################################
-#
-# test_file_name_parameters
-#
-#
-# Copyright (c) 11/9/2009 Leo Goodstadt
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#################################################################################
"""
test_file_name_parameters.py
@@ -32,17 +7,34 @@ from __future__ import print_function
"""
-import sys, os
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+import os
+import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = list(map(__import__, [ruffus_name]))[0]
+task = ruffus.task
+combine = ruffus.combine
+non_str_sequence = ruffus.file_name_parameters.non_str_sequence
+needs_update_check_modify_time = ruffus.file_name_parameters.needs_update_check_modify_time
+check_input_files_exist = ruffus.file_name_parameters.check_input_files_exist
+args_param_factory = ruffus.file_name_parameters.args_param_factory
+open_job_history = ruffus.file_name_parameters.open_job_history
+
+#from ruffus.file_name_parameters import *
-# Functions
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -52,10 +44,6 @@ sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-import sys
-import ruffus
-from ruffus import *
-from ruffus.file_name_parameters import *
#print ruffus.__version__
history_file = ':memory:'
history_file = None
@@ -192,13 +180,13 @@ l1 = [["input1", "output1.test"], [3, "output2.test"], ["input3", "output3.test"
l2 = [["output4.test", "output.ignored"]]
l3 = []
l4 = [[1, (2,"output5.test")]]
-t1 = task._task("module", "func1"); t1.param_generator_func = list_generator_factory(l1)
-t2 = task._task("module", "func2"); t2.param_generator_func = list_generator_factory(l2)
-t2._single_job_single_output = t2.single_job_single_output
-t3 = task._task("module", "func3"); t3.param_generator_func = list_generator_factory(l3)
-t4 = task._task("module", "func4"); t4.param_generator_func = list_generator_factory(l4)
-t4._single_job_single_output = t4.single_job_single_output
-t5 = task._task("module", "func5"); t5.param_generator_func = None
+t1 = task.Task(list_generator_factory, "module.func1"); t1.param_generator_func = list_generator_factory(l1)
+t2 = task.Task(list_generator_factory, "module.func2"); t2.param_generator_func = list_generator_factory(l2)
+t2._is_single_job_single_output = t2._single_job_single_output
+t3 = task.Task(list_generator_factory, "module.func3"); t3.param_generator_func = list_generator_factory(l3)
+t4 = task.Task(list_generator_factory, "module.func4"); t4.param_generator_func = list_generator_factory(l4)
+t4._is_single_job_single_output = t4._single_job_single_output
+t5 = task.Task(list_generator_factory, "module.func5"); t5.param_generator_func = None
next_task_id = 1
class Test_files_re_param_factory(unittest.TestCase):
@@ -229,43 +217,44 @@ class Test_files_re_param_factory(unittest.TestCase):
# self.assertRaises(ValueError, random.sample, self.seq, 20)
#def get_param_iterator (self, *old_args):
- def get_param_iterator (self, *orig_args):
+ def get_param_iterator (self, *unnamed_args, **named_args):
#
# replace function / function names with tasks
#
# fake virgin task
# use global incrementing index to avoid name clashes
- #fake_task = task._task("module", "func_fake%d" % randint(1, 1000000))
+ #fake_task = task.Task("module", "func_fake%d" % randint(1, 1000000))
global next_task_id
next_task_id += 1
- fake_task = task._task("module", "func_fake%d" % next_task_id)
- fake_task.task_files_re(orig_args)
+ fake_task = task.Task(list_generator_factory, "module.func_fake%d" % next_task_id)
+ fake_task._decorator_files_re(*unnamed_args, **named_args)
+ fake_task._complete_setup()
return fake_task.param_generator_func, fake_task
- def files_re (self, *old_args):
+ def files_re (self, *unnamed_args, **named_args):
"""
This extra function is to simulate the forwarding from the decorator to
the task creation function
"""
- return list(p1 for (p1, ps) in self.get_param_iterator(*old_args)[0](None))
+ return list(p1 for (p1, ps) in self.get_param_iterator(*unnamed_args, **named_args)[0](None))
#return list(self.get_param_iterator (*old_args)(None))
- def check_input_files_exist(self, *old_args):
+ def check_input_files_exist(self, *unnamed_args, **named_args):
"""
This extra function is to simulate the forwarding from the decorator to
the task creation function
"""
- it = self.get_param_iterator(*old_args)[0]
+ it = self.get_param_iterator(*unnamed_args, **named_args)[0]
for param, param2 in it(None):
check_input_files_exist (*param)
return True
- def needs_update_check_modify_time(self, *old_args):
+ def needs_update_check_modify_time(self, *unnamed_args, **named_args):
"""
This extra function is to simulate the forwarding from the decorator to
the task creation function
"""
- it, task = self.get_param_iterator(*old_args)
+ it, task = self.get_param_iterator(*unnamed_args, **named_args)
#print >> sys.stderr, [p for (p, param2) in it(None)], "??"
return [needs_update_check_modify_time (*p, task=task,
job_history = open_job_history(history_file)) for (p, param2) in it(None)]
@@ -448,24 +437,25 @@ class Test_split_param_factory(unittest.TestCase):
# wrappers
#_____________________________________________________________________________
- def get_param_iterator (self, *orig_args):
+ def get_param_iterator (self, *unnamed_args, **named_args):
#
# replace function / function names with tasks
#
# fake virgin task
- fake_task = task._task("module", "func_fake%d" % randint(1, 1000000))
- fake_task.task_split(orig_args)
+ fake_task = task.Task(list_generator_factory, "module.func_fake%d" % randint(1, 1000000))
+ fake_task._decorator_split(*unnamed_args, **named_args)
+ fake_task._complete_setup()
return fake_task.param_generator_func
- def do_task_split (self, *old_args):
+ def do_task_split (self, *unnamed_args, **named_args):
"""
This extra function is to simulate the forwarding from the decorator to
the task creation function
"""
# extra dereference because we are only interested in the first (only) job
- #return list(self.get_param_iterator (*old_args)(None))[0]
- return list(p1 for (p1, ps) in self.get_param_iterator (*old_args)(None))[0]
+ #return list(self.get_param_iterator (*unnamed_args, **named_args)(None))[0]
+ return list(p1 for (p1, ps) in self.get_param_iterator (*unnamed_args, **named_args)(None))[0]
@@ -583,22 +573,23 @@ class Test_merge_param_factory(unittest.TestCase):
# wrappers
#_____________________________________________________________________________
- def get_param_iterator (self, *orig_args):
+ def get_param_iterator (self, *unnamed_args, **named_args):
#
# replace function / function names with tasks
#
# fake virgin task
- fake_task = task._task("module", "func_fake%d" % randint(1, 1000000))
- fake_task.task_merge(orig_args)
+ fake_task = task.Task(list_generator_factory, "module.func_fake%d" % randint(1, 1000000))
+ fake_task._decorator_merge(*unnamed_args, **named_args)
+ fake_task._complete_setup()
return fake_task.param_generator_func
- def do_task_merge (self, *old_args):
+ def do_task_merge (self, *unnamed_args, **named_args):
"""
This extra function is to simulate the forwarding from the decorator to
the task creation function
"""
# extra dereference because we are only interested in the first (only) job
- return list(p1 for (p1, ps) in self.get_param_iterator (*old_args)(None))[0]
+ return list(p1 for (p1, ps) in self.get_param_iterator (*unnamed_args, **named_args)(None))[0]
@@ -606,6 +597,17 @@ class Test_merge_param_factory(unittest.TestCase):
"""
test globbed form
"""
+ expected_result = ( ['DIR/f0.output',
+ 'DIR/f0.test',
+ 'DIR/f1.output',
+ 'DIR/f1.test',
+ 'DIR/f2.output',
+ 'DIR/f2.test',
+ ],
+ ["test1",
+ "test2",
+ "extra.file"]
+ )
#
# simple 1 input, 1 output
#
@@ -614,24 +616,25 @@ class Test_merge_param_factory(unittest.TestCase):
"test2",
"extra.file"])
paths = recursive_replace(paths, test_path, "DIR")
- self.assertEqual(paths,
- ( ['DIR/f0.output',
- 'DIR/f0.test',
- 'DIR/f1.output',
- 'DIR/f1.test',
- 'DIR/f2.output',
- 'DIR/f2.test',
- ],
- ["test1",
- "test2",
- "extra.file"]
- ))
+ self.assertEqual(paths, expected_result)
+
+ #
+ # named parameters
+ #
+ paths = self.do_task_merge(input = test_path + "/*",
+ output = ["test1", # output params
+ "test2",
+ "extra.file"])
+ paths = recursive_replace(paths, test_path, "DIR")
+ self.assertEqual(paths, expected_result)
+
+
def test_tasks(self):
"""
test if can use tasks to specify dependencies
"""
- paths = self.do_task_merge([task.output_from("module.func1", # input params
+ unnamed_args = [ [task.output_from("module.func1", # input params
"module.func2",
"module.func3",
"module.func4",
@@ -640,10 +643,9 @@ class Test_merge_param_factory(unittest.TestCase):
["test1", # output params
"test2",
"extra.file"],
- 6) # extra params
- paths = recursive_replace(paths, test_path, "DIR")
- self.assertEqual(paths,
- ([
+ 6] # extra params
+
+ expected_results = ([
5,
['output4.test', 'output.ignored'],
'output1.test',
@@ -660,7 +662,20 @@ class Test_merge_param_factory(unittest.TestCase):
['test1', # output params
'test2',
'extra.file'],
- 6))
+ 6)
+
+ # unnamed arguments
+ paths = self.do_task_merge(*unnamed_args)
+ paths = recursive_replace(paths, test_path, "DIR")
+ self.assertEqual(paths, expected_results)
+
+ # NAMED ARGUMENTS
+ paths = self.do_task_merge(input = unnamed_args[0],
+ output = unnamed_args[1],
+ extras = unnamed_args[2:])
+ paths = recursive_replace(paths, test_path, "DIR")
+ self.assertEqual(paths, expected_results)
+
paths = self.do_task_merge(task.output_from("module.func2"), "output", "extra")
paths = self.do_task_merge(task.output_from("module.func1", "module.func2"), "output", "extra")
@@ -719,23 +734,24 @@ class Test_transform_param_factory(unittest.TestCase):
# wrappers
#_____________________________________________________________________________
- def get_param_iterator (self, *orig_args):
+ def get_param_iterator (self, *unnamed_args, **named_args):
#
# replace function / function names with tasks
#
# fake virgin task
- fake_task = task._task("module", "func_fake%d" % randint(1, 1000000))
- fake_task.task_transform(orig_args)
+ fake_task = task.Task(list_generator_factory, "module.func_fake%d" % randint(1, 1000000))
+ fake_task._decorator_transform(*unnamed_args, **named_args)
+ fake_task._complete_setup()
return fake_task.param_generator_func
- def do_task_transform (self, *old_args):
+ def do_task_transform (self, *unnamed_args, **named_args):
"""
This extra function is to simulate the forwarding from the decorator to
the task creation function
"""
# extra dereference because we are only interested in the first (only) job
- return list(p1 for (p1, ps) in self.get_param_iterator (*old_args)(None))
+ return list(p1 for (p1, ps) in self.get_param_iterator (*unnamed_args, **named_args)(None))
def test_simple(self):
@@ -775,18 +791,23 @@ class Test_transform_param_factory(unittest.TestCase):
#
# simple 1 input, 1 output
#
- paths = self.do_task_transform(test_path + "/*.test",
+ unnamed_args = [test_path + "/*.test",
task.formatter("/(?P<name>\w+).test$"),
- ["{path[0]}/{name[0]}.output1{ext[0]}", "{path[0]}/{name[0]}.output2"], "{path[0]}/{name[0]}.output3")
- #["{0[path][0]}"], ".txt")
-
- paths = recursive_replace(paths, test_path, "DIR")
- self.assertEqual(paths,
- [
+ ["{path[0]}/{name[0]}.output1{ext[0]}", "{path[0]}/{name[0]}.output2"], "{path[0]}/{name[0]}.output3"]
+ expected_results = [
('DIR/f0.test', ['DIR/f0.output1.test', 'DIR/f0.output2'], "DIR/f0.output3"),
('DIR/f1.test', ['DIR/f1.output1.test', 'DIR/f1.output2'], "DIR/f1.output3"),
- ('DIR/f2.test', ['DIR/f2.output1.test', 'DIR/f2.output2'], "DIR/f2.output3"),
- ])
+ ('DIR/f2.test', ['DIR/f2.output1.test', 'DIR/f2.output2'], "DIR/f2.output3"),]
+ # unnamed_args ]
+ paths = self.do_task_transform(*unnamed_args)
+ paths = recursive_replace(paths, test_path, "DIR")
+ self.assertEqual(paths, expected_results)
+
+ #named args
+ paths = self.do_task_transform(input = unnamed_args[0], filter = unnamed_args[1], output = unnamed_args[2], extras=unnamed_args[3:])
+ paths = recursive_replace(paths, test_path, "DIR")
+ self.assertEqual(paths, expected_results)
+
def test_regex(self):
"""
test regex transform with globs
@@ -794,16 +815,25 @@ class Test_transform_param_factory(unittest.TestCase):
#
# simple 1 input, 1 output
#
- paths = self.do_task_transform(test_path + "/*.test", task.regex(r"(.*)\.test"),
- [r"\1.output1", r"\1.output2"], r"\1.output3")
-
- paths = recursive_replace(paths, test_path, "DIR")
- self.assertEqual(paths,
- [
+ unnamed_args = [test_path + "/*.test",
+ task.regex(r"(.*)\.test"),
+ [r"\1.output1", r"\1.output2"], r"\1.output3"]
+ expected_results = [
('DIR/f0.test', ['DIR/f0.output1', 'DIR/f0.output2'], "DIR/f0.output3"),
('DIR/f1.test', ['DIR/f1.output1', 'DIR/f1.output2'], "DIR/f1.output3"),
- ('DIR/f2.test', ['DIR/f2.output1', 'DIR/f2.output2'], "DIR/f2.output3"),
- ])
+ ('DIR/f2.test', ['DIR/f2.output1', 'DIR/f2.output2'], "DIR/f2.output3"),]
+
+
+ # unnamed_args ]
+ paths = self.do_task_transform(*unnamed_args)
+ paths = recursive_replace(paths, test_path, "DIR")
+ self.assertEqual(paths, expected_results)
+
+ #named args
+ paths = self.do_task_transform(input = unnamed_args[0], filter = unnamed_args[1], output = unnamed_args[2], extras=unnamed_args[3:])
+ paths = recursive_replace(paths, test_path, "DIR")
+ self.assertEqual(paths, expected_results)
+
def test_inputs(self):
"""
@@ -838,17 +868,33 @@ class Test_transform_param_factory(unittest.TestCase):
# add inputs
#
#
- paths = self.do_task_transform(test_path + "/*.test", task.regex(r"(.*)\.test"),
+ unnamed_args = [test_path + "/*.test", task.regex(r"(.*)\.test"),
task.add_inputs(r"\1.testwhat"),
- [r"\1.output1", r"\1.output2"])
-
- paths = recursive_replace(paths, test_path, "DIR")
- self.assertEqual(paths,
- [
+ [r"\1.output1", r"\1.output2"]]
+ expected_results = [
(('DIR/f0.test','DIR/f0.testwhat'), ['DIR/f0.output1', 'DIR/f0.output2']),
(('DIR/f1.test','DIR/f1.testwhat'), ['DIR/f1.output1', 'DIR/f1.output2']),
(('DIR/f2.test','DIR/f2.testwhat'), ['DIR/f2.output1', 'DIR/f2.output2']),
- ])
+ ]
+
+ # unnamed_args ]
+ paths = self.do_task_transform(*unnamed_args)
+ paths = recursive_replace(paths, test_path, "DIR")
+ self.assertEqual(paths, expected_results)
+
+ #named args
+ paths = self.do_task_transform(input = unnamed_args[0], filter = unnamed_args[1], add_inputs = unnamed_args[2], output = unnamed_args[3], extras=unnamed_args[4:])
+ paths = recursive_replace(paths, test_path, "DIR")
+ self.assertEqual(paths, expected_results)
+
+ #named args
+ paths = self.do_task_transform(*unnamed_args[0:2], add_inputs = unnamed_args[2].args, output = unnamed_args[3], extras=unnamed_args[4:])
+ paths = recursive_replace(paths, test_path, "DIR")
+ self.assertEqual(paths, expected_results)
+
+
+
+
paths = self.do_task_transform(test_path + "/*.test", task.suffix(".test"),
task.add_inputs(r"a.testwhat"),
[".output1", ".output2"], ".output3")
@@ -954,23 +1000,24 @@ class Test_collate_param_factory(unittest.TestCase):
# wrappers
#_____________________________________________________________________________
- def get_param_iterator (self, *orig_args):
+ def get_param_iterator (self, *unnamed_args, **named_args):
#
# replace function / function names with tasks
#
# fake virgin task
- fake_task = task._task("module", "func_fake%d" % randint(1, 1000000))
- fake_task.task_collate(orig_args)
+ fake_task = task.Task(list_generator_factory, "module.func_fake%d" % randint(1, 1000000))
+ fake_task._decorator_collate(*unnamed_args, **named_args)
+ fake_task._complete_setup()
return fake_task.param_generator_func
- def do_task_collate (self, *old_args):
+ def do_task_collate (self, *unnamed_args, **named_args):
"""
This extra function is to simulate the forwarding from the decorator to
the task creation function
"""
# extra dereference because we are only interested in the first (only) job
- return list(p1 for (p1, ps) in self.get_param_iterator (*old_args)(None))
+ return list(p1 for (p1, ps) in self.get_param_iterator (*unnamed_args, **named_args)(None))
def test_regex(self):
@@ -1078,12 +1125,13 @@ class Test_collate_param_factory(unittest.TestCase):
#
# test python set object. Note that set is constructed with the results of the substitution
#
- paths = self.do_task_collate(test_path + "/*.test", task.regex(r"(.*/[ef])[a-z0-9]+\.test"),
- task.inputs(r"\1.whoopee"), set([r"\1.output1", r"\1.output2", test_path + "/e.output2"]), r"\1.extra")
- paths = recursive_replace(paths, test_path, "DIR")
- self.assertEqual(paths,
- [
+ unnamed_args = [test_path + "/*.test",
+ task.regex(r"(.*/[ef])[a-z0-9]+\.test"),
+ task.inputs(r"\1.whoopee"),
+ set([r"\1.output1", r"\1.output2", test_path + "/e.output2"]),
+ r"\1.extra"]
+ expected_results = [
(
('DIR/e.whoopee',), # input
set(['DIR/e.output1', 'DIR/e.output2']), # output
@@ -1094,7 +1142,28 @@ class Test_collate_param_factory(unittest.TestCase):
set(['DIR/f.output1', 'DIR/f.output2', 'DIR/e.output2']), # output
'DIR/f.extra' # extra
)
- ])
+ ]
+
+ # unnamed_args ]
+ paths = self.do_task_collate(*unnamed_args)
+ paths = recursive_replace(paths, test_path, "DIR")
+ self.assertEqual(paths, expected_results)
+
+ #named args
+ paths = self.do_task_collate(input = unnamed_args[0], filter = unnamed_args[1], replace_inputs = unnamed_args[2], output = unnamed_args[3], extras=unnamed_args[4:])
+ paths = recursive_replace(paths, test_path, "DIR")
+ self.assertEqual(paths, expected_results)
+
+ #named args
+ paths = self.do_task_collate(*unnamed_args[0:2], replace_inputs = unnamed_args[2], output = unnamed_args[3], extras=unnamed_args[4:])
+ paths = recursive_replace(paths, test_path, "DIR")
+ self.assertEqual(paths, expected_results)
+
+ #named args
+ paths = self.do_task_collate(*unnamed_args[0:2], replace_inputs = unnamed_args[2].args[0], output = unnamed_args[3], extras=unnamed_args[4:])
+ paths = recursive_replace(paths, test_path, "DIR")
+ self.assertEqual(paths, expected_results)
+
def test_tasks(self):
"""
@@ -1201,21 +1270,22 @@ class Test_files_param_factory(unittest.TestCase):
pass
- def get_param_iterator (self, *orig_args):
+ def get_param_iterator (self, *unnamed_args, **named_args):
#
# replace function / function names with tasks
#
# fake virgin task
- fake_task = task._task("module", "func_fake%d" % randint(1, 1000000))
- fake_task.task_files(orig_args)
+ fake_task = task.Task(list_generator_factory, "module.func_fake%d" % randint(1, 1000000))
+ fake_task._decorator_files(*unnamed_args, **named_args)
+ fake_task._complete_setup()
return fake_task.param_generator_func
- def files (self, *old_args):
+ def files (self, *unnamed_args, **named_args):
"""
This extra function is to simulate the forwarding from the decorator to
the task creation function
"""
- return list(p1 for (p1, ps) in self.get_param_iterator (*old_args)(None))
+ return list(p1 for (p1, ps) in self.get_param_iterator (*unnamed_args, **named_args)(None))
def _test_simple(self):
"""
@@ -1425,23 +1495,24 @@ class Test_product_param_factory(unittest.TestCase):
# wrappers
#_____________________________________________________________________________
- def get_param_iterator (self, *orig_args):
+ def get_param_iterator (self, *unnamed_args, **named_args):
#
# replace function / function names with tasks
#
# fake virgin task
- fake_task = task._task("module", "func_fake%d" % randint(1, 1000000))
- fake_task.task_product(orig_args)
+ fake_task = task.Task(list_generator_factory, "module.func_fake%d" % randint(1, 1000000))
+ fake_task._decorator_product(*unnamed_args, **named_args)
+ fake_task._complete_setup()
return fake_task.param_generator_func
- def do_task_product (self, *old_args):
+ def do_task_product (self, *unnamed_args, **named_args):
"""
This extra function is to simulate the forwarding from the decorator to
the task creation function
"""
# extra dereference because we are only interested in the first (only) job
- return list(p1 for (p1, ps) in self.get_param_iterator (*old_args)(None))
+ return list(p1 for (p1, ps) in self.get_param_iterator (*unnamed_args, **named_args)(None))
def test_simple(self):
@@ -1451,18 +1522,36 @@ class Test_product_param_factory(unittest.TestCase):
#
# simple 1 input, 1 output
#
- paths = self.do_task_product([test_path + "/a.test1", test_path + "/b.test1"], task.formatter("(?P<ID>\w+)\.(.+)"),
- [test_path + "/c.test2", test_path + "/d.test2", test_path + "/e.ignore"], task.formatter("(?P<ID>\w+)\.(test2)"),
- r"{path[0][0]}/{ID[0][0]}.{1[1][0]}.output")
-
- self.assertEqual(recursive_replace(paths, test_path, "DIR"),
- [
+ args = [[test_path + "/a.test1", test_path + "/b.test1"],
+ task.formatter("(?:.+/)?(?P<ID>\w+)\.(.+)"),
+ [test_path + "/c.test2", test_path + "/d.test2", test_path + "/e.ignore"],
+ task.formatter("(?:.+/)?(?P<ID>\w+)\.(test2)"),
+ r"{path[0][0]}/{ID[0][0]}.{1[1][0]}.output"]
+ expected_result = [
(('DIR/a.test1','DIR/c.test2'),'DIR/a.c.output'),
(('DIR/a.test1','DIR/d.test2'),'DIR/a.d.output'),
(('DIR/b.test1','DIR/c.test2'),'DIR/b.c.output'),
(('DIR/b.test1','DIR/d.test2'),'DIR/b.d.output')
]
- )
+ paths = self.do_task_product(*args)
+ self.assertEqual(recursive_replace(paths, test_path, "DIR"), expected_result)
+
+ # named parameters
+ paths = self.do_task_product(input = args[0], filter = args[1],
+ input2 = args[2], filter2 = args[3],
+ output = args [4])
+ self.assertEqual(recursive_replace(paths, test_path, "DIR"), expected_result)
+
+ # named parameters
+ paths = self.do_task_product(*args[0:2],
+ input2 = args[2], filter2 = args[3],
+ output = args [4])
+ self.assertEqual(recursive_replace(paths, test_path, "DIR"), expected_result)
+
+ paths = self.do_task_product(*args[0:4],
+ output = args [4])
+ self.assertEqual(recursive_replace(paths, test_path, "DIR"), expected_result)
+
def test_inputs(self):
"""
@@ -1472,8 +1561,8 @@ class Test_product_param_factory(unittest.TestCase):
# (replace) inputs
#
#
- paths = self.do_task_product([test_path + "/a.test1", test_path + "/b.test1"], task.formatter("(?P<ID>\w+)\.(.+)"),
- [test_path + "/c.test2", test_path + "/d.test2", test_path + "/e.ignore"], task.formatter("(?P<ID>\w+)\.(test2)"),
+ paths = self.do_task_product([test_path + "/a.test1", test_path + "/b.test1"], task.formatter("(?:.+/)?(?P<ID>\w+)\.(.+)"),
+ [test_path + "/c.test2", test_path + "/d.test2", test_path + "/e.ignore"], task.formatter("(?:.+/)?(?P<ID>\w+)\.(test2)"),
task.inputs(("{path[0][0]}/{basename[0][0]}.testwhat1", "{path[1][0]}/{basename[1][0]}.testwhat2") ),
r"{path[0][0]}/{ID[0][0]}.{1[1][0]}.output")
paths = recursive_replace(paths, test_path, "DIR")
@@ -1489,8 +1578,8 @@ class Test_product_param_factory(unittest.TestCase):
# add inputs
#
#
- paths = self.do_task_product([test_path + "/a.test1", test_path + "/b.test1"], task.formatter("(?P<ID>\w+)\.(.+)"),
- [test_path + "/c.test2", test_path + "/d.test2", test_path + "/e.ignore"], task.formatter("(?P<ID>\w+)\.(test2)"),
+ paths = self.do_task_product([test_path + "/a.test1", test_path + "/b.test1"], task.formatter("(?:.+/)?(?P<ID>\w+)\.(.+)"),
+ [test_path + "/c.test2", test_path + "/d.test2", test_path + "/e.ignore"], task.formatter("(?:.+/)?(?P<ID>\w+)\.(test2)"),
task.add_inputs("{path[0][0]}/{basename[0][0]}.testwhat1", "{path[1][0]}/{basename[1][0]}.testwhat2", ),
r"{path[0][0]}/{ID[0][0]}.{1[1][0]}.output")
@@ -1506,13 +1595,5 @@ class Test_product_param_factory(unittest.TestCase):
-#
-# debug parameter ignored if called as a module
-#
-if sys.argv.count("--debug"):
- sys.argv.remove("--debug")
-#sys.argv.append("Test_files_param_factory")
-#sys.argv.append("Test_merge_param_factory")
-#sys.argv.append("Test_transform_param_factory")
-#sys.argv.append("Test_files_param_factory")
-unittest.main()
+if __name__ == '__main__':
+ unittest.main()
diff --git a/ruffus/test/test_files_decorator.py b/ruffus/test/test_files_decorator.py
index 980f069..2edbee1 100755
--- a/ruffus/test/test_files_decorator.py
+++ b/ruffus/test/test_files_decorator.py
@@ -9,134 +9,56 @@ from __future__ import print_function
"""
+import os
+import sys
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
-# options
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
-from optparse import OptionParser
-import sys, os
-import os.path
try:
- import StringIO as io
-except:
- import io as io
-
-import re,time
-
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-if __name__ == '__main__':
- module_name = os.path.split(sys.argv[0])[1]
- module_name = os.path.splitext(module_name)[0];
-else:
- module_name = __name__
-
-
-
-import ruffus
-print("\tRuffus Version = ", ruffus.__version__)
-parser = OptionParser(version="%%prog v1.0, ruffus v%s" % ruffus.ruffus_version.__version)
-parser.add_option("-t", "--target_tasks", dest="target_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Target task(s) of pipeline.")
-parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Pipeline task(s) which will be included even if they are up to date.")
-parser.add_option("-j", "--jobs", dest="jobs",
- default=1,
- metavar="jobs",
- type="int",
- help="Specifies the number of jobs (commands) to run simultaneously.")
-parser.add_option("-v", "--verbose", dest = "verbose",
- action="count", default=0,
- help="Print more verbose messages for each additional verbose level.")
-parser.add_option("-d", "--dependency", dest="dependency_file",
- #default="simple.svg",
- metavar="FILE",
- type="string",
- help="Print a dependency graph of the pipeline that would be executed "
- "to FILE, but do not execute it.")
-parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
- metavar="FORMAT",
- type="string",
- default = 'svg',
- help="format of dependency graph file. Can be 'ps' (PostScript), "+
- "'svg' 'svgz' (Structured Vector Graphics), " +
- "'png' 'gif' (bitmap graphics) etc ")
-parser.add_option("-n", "--just_print", dest="just_print",
- action="store_true", default=False,
- help="Print a description of the jobs that would be executed, "
- "but do not execute them.")
-parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
- action="store_true", default=False,
- help="Do not print out legend and key for dependency graph.")
-parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
- action="store_true", default=False,
- help="Draw horizontal dependency graph.")
-
-parameters = [
- ]
-
-
-
-
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
+try:
+ attrlist = ruffus.file_name_parameters.__all__
+except AttributeError:
+ attrlist = dir (ruffus.file_name_parameters)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus.file_name_parameters, attr)
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-# imports
+# imports
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-import re
-import operator
-import sys,os
-from collections import defaultdict
-import random
-
-sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-
+import json
# use simplejson in place of json for python < 2.6
-try:
- import json
-except ImportError:
- import simplejson
- json = simplejson
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# Main logic
-
+#try:
+#except ImportError:
+# import simplejson
+# json = simplejson
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-
-
-
-# get help string
-f =io.StringIO()
-parser.print_help(f)
-helpstr = f.getvalue()
-(options, remaining_args) = parser.parse_args()
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# Functions
+# Tasks
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -151,11 +73,11 @@ def test_job_io(infiles, outfiles, extra_params):
if isinstance(infiles, str):
infile_names = [infiles]
- elif infiles == None:
+ elif infiles is None:
infile_names = []
else:
infile_names = infiles
-
+
if isinstance(outfiles, str):
outfile_names = [outfiles]
else:
@@ -172,8 +94,8 @@ def test_job_io(infiles, outfiles, extra_params):
with open(f, "w") as oo:
oo.write(output_text)
-
-
+
+
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
# Tasks
@@ -254,24 +176,26 @@ class Test_task(unittest.TestCase):
def test_task (self):
- pipeline_run([task5], options.forced_tasks, multiprocess = options.jobs,
- verbose = options.verbose)
+ pipeline_run(multiprocess = 10, verbose = 0)
+
+ def test_newstyle_task (self):
+ """
+ Same as above but construct a new pipeline on the fly without decorators
+ """
+ test_pipeline = Pipeline("test")
+ test_pipeline.files(task1, None, tempdir + 'a.1')\
+ .follows(mkdir(tempdir))
+ test_pipeline.transform(task_func = task2,
+ input = task1,
+ filter = regex(r".*"),
+ output = tempdir + 'b.1')
+ test_pipeline.files(task3, task2, tempdir + 'c.1')
+ test_pipeline.files(task4, [[None, tempdir + 'd.1'], [None, tempdir + 'e.1']])\
+ .follows(task3)
+ test_pipeline.files(task5, task4, tempdir + "f.1")
+ test_pipeline.run(multiprocess = 10, verbose = 0)
if __name__ == '__main__':
- if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
- verbose = options.verbose)
-
- elif options.dependency_file:
- with open(options.dependency_file, "w") as graph_file:
- pipeline_printout_graph (graph_file,
- options.dependency_graph_format,
- options.target_tasks,
- options.forced_tasks,
- draw_vertically = not options.draw_horizontally,
- no_key_legend = options.no_key_legend_in_graph)
- else:
- sys.argv= sys.argv[0:1]
- unittest.main()
-
+ unittest.main()
+
diff --git a/ruffus/test/test_files_post_merge.py b/ruffus/test/test_files_post_merge.py
deleted file mode 100755
index 4a9903c..0000000
--- a/ruffus/test/test_files_post_merge.py
+++ /dev/null
@@ -1,299 +0,0 @@
-#!/usr/bin/env python
-from __future__ import print_function
-"""
-
- test_files_post_merge.py
-
- bug where @files follows merge and extra parenthesis inserted
-
- use :
- --debug to test automatically
- --start_again the first time you run the file
- --jobs_per_task N to simulate tasks with N numbers of files per task
-
- -j N / --jobs N to speify multitasking
- -v to see the jobs in action
- -n / --just_print to see what jobs would run
-
-"""
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# options
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-from optparse import OptionParser
-import sys, os
-import os.path
-try:
- import StringIO as io
-except:
- import io as io
-
-import re,time
-
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-if __name__ == '__main__':
- module_name = os.path.split(sys.argv[0])[1]
- module_name = os.path.splitext(module_name)[0];
-else:
- module_name = __name__
-
-
-
-import ruffus
-parser = OptionParser(version="%%prog v1.0, ruffus v%s" % ruffus.ruffus_version.__version)
-parser.add_option("-D", "--debug", dest="debug",
- action="store_true", default=False,
- help="Make sure output is correct and clean up.")
-parser.add_option("-s", "--start_again", dest="start_again",
- action="store_true", default=False,
- help="Make a new 'original.fa' file to simulate having to restart "
- "pipeline from scratch.")
-parser.add_option("--jobs_per_task", dest="jobs_per_task",
- default=3,
- metavar="N",
- type="int",
- help="Simulates tasks with N numbers of files per task.")
-
-
-parser.add_option("-t", "--target_tasks", dest="target_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Target task(s) of pipeline.")
-parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Pipeline task(s) which will be included even if they are up to date.")
-parser.add_option("-j", "--jobs", dest="jobs",
- default=1,
- metavar="jobs",
- type="int",
- help="Specifies the number of jobs (commands) to run simultaneously.")
-parser.add_option("-v", "--verbose", dest = "verbose",
- action="count", default=0,
- help="Print more verbose messages for each additional verbose level.")
-parser.add_option("-d", "--dependency", dest="dependency_file",
- #default="simple.svg",
- metavar="FILE",
- type="string",
- help="Print a dependency graph of the pipeline that would be executed "
- "to FILE, but do not execute it.")
-parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
- metavar="FORMAT",
- type="string",
- default = 'svg',
- help="format of dependency graph file. Can be 'ps' (PostScript), "+
- "'svg' 'svgz' (Structured Vector Graphics), " +
- "'png' 'gif' (bitmap graphics) etc ")
-parser.add_option("-n", "--just_print", dest="just_print",
- action="store_true", default=False,
- help="Print a description of the jobs that would be executed, "
- "but do not execute them.")
-parser.add_option("-M", "--minimal_rebuild_mode", dest="minimal_rebuild_mode",
- action="store_true", default=False,
- help="Rebuild a minimum of tasks necessary for the target. "
- "Ignore upstream out of date tasks if intervening tasks are fine.")
-parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
- action="store_true", default=False,
- help="Do not print out legend and key for dependency graph.")
-parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
- action="store_true", default=False,
- help="Draw horizontal dependency graph.")
-
-parameters = [
- ]
-
-
-
-
-
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# imports
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-
-import re
-import operator
-import sys,os
-from collections import defaultdict
-import random
-
-sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-
-# use simplejson in place of json for python < 2.6
-try:
- import json
-except ImportError:
- import simplejson
- json = simplejson
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# Main logic
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-
-
-
-
-# get help string
-f =io.StringIO()
-parser.print_help(f)
-helpstr = f.getvalue()
-(options, remaining_args) = parser.parse_args()
-
-
-tempdir = "temp_filesre_split_and_combine/"
-
-
-
-if options.verbose:
- verbose_output = sys.stderr
-else:
- verbose_output =open("/dev/null", "w")
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# Tasks
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-#
-# split_fasta_file
-#
- at posttask(lambda: verbose_output.write("Split into %d files\n" % options.jobs_per_task))
- at split(tempdir + "original.fa", [tempdir + "files.split.success", tempdir + "files.split.*.fa"])
-def split_fasta_file (input_file, outputs):
-
- #
- # remove previous fasta files
- #
- success_flag = outputs[0]
- output_file_names = outputs[1:]
- for f in output_file_names:
- os.unlink(f)
-
- #
- # create as many files as we are simulating in jobs_per_task
- #
- for i in range(options.jobs_per_task):
- open(tempdir + "files.split.%03d.fa" % i, "w")
-
- open(success_flag, "w")
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-#
-# align_sequences
-#
- at posttask(lambda: verbose_output.write("Sequences aligned\n"))
- at transform(split_fasta_file, suffix(".fa"), ".aln") # fa -> aln
-def align_sequences (input_file, output_filename):
- open(output_filename, "w").write("%s\n" % output_filename)
-
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-#
-# percentage_identity
-#
- at posttask(lambda: verbose_output.write("%Identity calculated\n"))
- at transform(align_sequences, # find all results from align_sequences
- suffix(".aln"), # replace suffix with:
- [r".pcid", # .pcid suffix for the result
- r".pcid_success"]) # .pcid_success to indicate job completed
-def percentage_identity (input_file, output_files):
- (output_filename, success_flag_filename) = output_files
- open(output_filename, "w").write("%s\n" % output_filename)
- open(success_flag_filename, "w")
-
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-#
-# combine_results
-#
- at posttask(lambda: verbose_output.write("Results recombined\n"))
- at merge(percentage_identity, tempdir + "all.combine_results")
-def combine_results (input_files, output_files):
- """
- Combine all
- """
- (output_filename) = output_files
- out = open(output_filename, "w")
- for inp, flag in input_files:
- out.write(open(inp).read())
-
-
-
- at files(combine_results, os.path.join(tempdir, "check_all_is.well"))
-def post_merge_check (input_filename, output_filename):
- """
- check that merge sends just one file, not a list to me
- """
- open(output_filename, "w").write(open(input_filename).read())
-
- at files(post_merge_check, os.path.join(tempdir, "check_all_is.weller"))
-def post_post_merge_check (input_filename, output_filename):
- """
- check that @files forwards a single file on when given a single file
- """
- open(output_filename, "w").write(open(input_filename).read())
-
-def start_pipeline_afresh ():
- """
- Recreate directory and starting file
- """
- print("Start again", file=verbose_output)
- import os
- os.system("rm -rf %s" % tempdir)
- os.makedirs(tempdir)
- open(tempdir + "original.fa", "w").close()
-
-if __name__ == '__main__':
- if options.start_again:
- start_pipeline_afresh()
- if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
- verbose = options.verbose,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
-
- elif options.dependency_file:
- pipeline_printout_graph ( open(options.dependency_file, "w"),
- options.dependency_graph_format,
- options.target_tasks,
- options.forced_tasks,
- draw_vertically = not options.draw_horizontally,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- no_key_legend = options.no_key_legend_in_graph)
- elif options.debug:
- start_pipeline_afresh()
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- logger = stderr_logger if options.verbose else black_hole_logger,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- verbose = options.verbose)
- os.system("rm -rf %s" % tempdir)
- print("OK")
- else:
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- logger = stderr_logger if options.verbose else black_hole_logger,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- verbose = options.verbose)
-
diff --git a/ruffus/test/test_filesre_combine.py b/ruffus/test/test_filesre_combine.py
index a0e15fb..2d938fc 100755
--- a/ruffus/test/test_filesre_combine.py
+++ b/ruffus/test/test_filesre_combine.py
@@ -8,87 +8,29 @@ from __future__ import print_function
"""
+import os
+import sys
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
-# options
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
-from optparse import OptionParser
-import sys, os
-import os.path
try:
- import StringIO as io
-except:
- import io as io
-
-import re
-
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-if __name__ == '__main__':
- module_name = os.path.split(sys.argv[0])[1]
- module_name = os.path.splitext(module_name)[0];
-else:
- module_name = __name__
-
-
-
-
-parser = OptionParser(version="%prog 1.0")
-parser.add_option("-D", "--debug", dest="debug",
- action="store_true", default=False,
- help="Make sure output is correct and clean up.")
-parser.add_option("-t", "--target_tasks", dest="target_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Target task(s) of pipeline.")
-parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Pipeline task(s) which will be included even if they are up to date.")
-parser.add_option("-j", "--jobs", dest="jobs",
- default=1,
- metavar="jobs",
- type="int",
- help="Specifies the number of jobs (commands) to run simultaneously.")
-parser.add_option("-v", "--verbose", dest = "verbose",
- action="count", default=0,
- help="Do not echo to shell but only print to log.")
-parser.add_option("-F", "--flowchart", dest="flowchart",
- #default="simple.svg",
- metavar="FILE",
- type="string",
- help="Print a flowchart of the pipeline that would be executed "
- "to FILE, but do not execute it.")
-parser.add_option("--flowchart_format", dest="flowchart_format",
- metavar="FORMAT",
- type="string",
- default = 'svg',
- help="format of flowchart file. Can be 'ps' (PostScript), "+
- "'svg' 'svgz' (Structured Vector Graphics), " +
- "'png' 'gif' (bitmap graphics) etc ")
-parser.add_option("-n", "--just_print", dest="just_print",
- action="store_true", default=False,
- help="Print a description of the jobs that would be executed, "
- "but do not execute them.")
-parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
- action="store_true", default=False,
- help="Do not print out legend and key for dependency graph.")
-parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
- action="store_true", default=False,
- help="Draw horizontal dependency graph.")
-
-parameters = [
- ]
-
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
@@ -102,21 +44,15 @@ parameters = [
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-import re
-import operator
-import sys,os
from collections import defaultdict
-import random
-
-sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
+import json
# use simplejson in place of json for python < 2.6
-try:
- import json
-except ImportError:
- import simplejson
- json = simplejson
+#try:
+# import json
+#except ImportError:
+# import simplejson
+# json = simplejson
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -129,12 +65,6 @@ except ImportError:
-# get help string
-f =io.StringIO()
-parser.print_help(f)
-helpstr = f.getvalue()
-(options, remaining_args) = parser.parse_args()
-
species_list = defaultdict(list)
species_list["mammals"].append("cow" )
species_list["mammals"].append("horse" )
@@ -146,6 +76,10 @@ species_list["fish" ].append("pufferfish")
tempdir = "temp_filesre_combine/"
+def do_write(file_name, what):
+ with open(file_name, "a") as oo:
+ oo.write(what)
+test_file = tempdir + "task.done"
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -158,12 +92,13 @@ tempdir = "temp_filesre_combine/"
# task1
#
@follows(mkdir(tempdir, tempdir + "test"))
- at posttask(lambda: open(tempdir + "task.done", "a").write("Task 1 Done\n"))
+ at posttask(lambda: do_write(test_file, "Task 1 Done\n"))
def prepare_files ():
for grouping in species_list.keys():
for species_name in species_list[grouping]:
filename = tempdir + "%s.%s.animal" % (species_name, grouping)
- open(filename, "w").write(species_name + "\n")
+ with open(filename, "w") as oo:
+ oo.write(species_name + "\n")
#
@@ -171,16 +106,19 @@ def prepare_files ():
#
@files_re(tempdir + '*.animal', r'(.*/)(.*)\.(.*)\.animal', combine(r'\1\2.\3.animal'), r'\1\3.results')
@follows(prepare_files)
- at posttask(lambda: open(tempdir + "task.done", "a").write("Task 2 Done\n"))
+ at posttask(lambda: do_write(test_file, "Task 2 Done\n"))
def summarise_by_grouping(infiles, outfile):
"""
Summarise by each species group, e.g. mammals, reptiles, fish
"""
- open(tempdir + "jobs.start", "a").write('job = %s\n' % json.dumps([infiles, outfile]))
- o = open(outfile, "w")
- for i in infiles:
- o.write(open(i).read())
- open(tempdir + "jobs.finish", "a").write('job = %s\n' % json.dumps([infiles, outfile]))
+ with open(tempdir + "jobs.start", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfile]))
+ with open(outfile, "w") as oo:
+ for i in infiles:
+ with open(i) as ii:
+ oo.write(ii.read())
+ with open(tempdir + "jobs.finish", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfile]))
@@ -204,39 +142,34 @@ def check_species_correct():
# -> fish.results
"""
for grouping in species_list:
- assert(open(tempdir + grouping + ".results").read() ==
- "".join(s + "\n" for s in sorted(species_list[grouping])))
+ with open(tempdir + grouping + ".results") as ii:
+ assert(ii.read() ==
+ "".join(s + "\n" for s in sorted(species_list[grouping])))
-#
-# Necessary to protect the "entry point" of the program under windows.
-# see: http://docs.python.org/library/multiprocessing.html#multiprocessing-programming
-#
-if __name__ == '__main__':
- if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
- verbose=options.verbose)
-
- elif options.flowchart:
- pipeline_printout_graph ( open(options.flowchart, "w"),
- options.flowchart_format,
- options.target_tasks,
- options.forced_tasks,
- draw_vertically = not options.draw_horizontally,
- no_key_legend = options.no_key_legend_in_graph)
- elif options.debug:
- import os
- os.system("rm -rf %s" % tempdir)
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- verbose = options.verbose)
+import unittest, shutil
+class Test_ruffus(unittest.TestCase):
+ def tearDown(self):
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+ def setUp(self):
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+ def test_ruffus (self):
+ ""
+ pipeline_run(multiprocess = 10, verbose = 0)
check_species_correct()
- os.system("rm -rf %s" % tempdir)
- print("OK")
- else:
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- verbose = options.verbose)
+
+
+if __name__ == '__main__':
+ unittest.main()
+
diff --git a/ruffus/test/test_filesre_split_and_combine.py b/ruffus/test/test_filesre_split_and_combine.py
index 97121e6..6c074b2 100755
--- a/ruffus/test/test_filesre_split_and_combine.py
+++ b/ruffus/test/test_filesre_split_and_combine.py
@@ -18,157 +18,63 @@ from __future__ import print_function
"""
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# options
+import os
+import sys
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-from optparse import OptionParser
-import sys, os
-import os.path
-try:
- import StringIO as io
-except:
- import io as io
-import re,time
-
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-if __name__ == '__main__':
- module_name = os.path.split(sys.argv[0])[1]
- module_name = os.path.splitext(module_name)[0];
-else:
- module_name = __name__
-
-
-
-import ruffus
-parser = OptionParser(version="%%prog v1.0, ruffus v%s" % ruffus.ruffus_version.__version)
-parser.add_option("-D", "--debug", dest="debug",
- action="store_true", default=False,
- help="Make sure output is correct and clean up.")
-parser.add_option("-s", "--start_again", dest="start_again",
- action="store_true", default=False,
- help="Make a new 'original.fa' file to simulate having to restart "
- "pipeline from scratch.")
-parser.add_option("--jobs_per_task", dest="jobs_per_task",
- default=50,
- metavar="N",
- type="int",
- help="Simulates tasks with N numbers of files per task.")
-
-
-parser.add_option("-t", "--target_tasks", dest="target_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Target task(s) of pipeline.")
-parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Pipeline task(s) which will be included even if they are up to date.")
-parser.add_option("-j", "--jobs", dest="jobs",
- default=1,
- metavar="jobs",
- type="int",
- help="Specifies the number of jobs (commands) to run simultaneously.")
-parser.add_option("-v", "--verbose", dest = "verbose",
- action="count", default=0,
- help="Print more verbose messages for each additional verbose level.")
-parser.add_option("-d", "--dependency", dest="dependency_file",
- #default="simple.svg",
- metavar="FILE",
- type="string",
- help="Print a dependency graph of the pipeline that would be executed "
- "to FILE, but do not execute it.")
-parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
- metavar="FORMAT",
- type="string",
- default = 'svg',
- help="format of dependency graph file. Can be 'ps' (PostScript), "+
- "'svg' 'svgz' (Structured Vector Graphics), " +
- "'png' 'gif' (bitmap graphics) etc ")
-parser.add_option("-n", "--just_print", dest="just_print",
- action="store_true", default=False,
- help="Print a description of the jobs that would be executed, "
- "but do not execute them.")
-parser.add_option("-M", "--minimal_rebuild_mode", dest="minimal_rebuild_mode",
- action="store_true", default=False,
- help="Rebuild a minimum of tasks necessary for the target. "
- "Ignore upstream out of date tasks if intervening tasks are fine.")
-parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
- action="store_true", default=False,
- help="Do not print out legend and key for dependency graph.")
-parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
- action="store_true", default=False,
- help="Draw horizontal dependency graph.")
-
-parameters = [
- ]
-
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+try:
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-# imports
+# constants
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-import re
-import operator
-import sys,os
-from collections import defaultdict
-import random
-
-sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-
-# use simplejson in place of json for python < 2.6
-try:
- import json
-except ImportError:
- import simplejson
- json = simplejson
+JOBS_PER_TASK = 50
+tempdir = "temp_filesre_split_and_combine/"
+verbose_output = sys.stderr
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-# Main logic
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# imports
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# get help string
-f =io.StringIO()
-parser.print_help(f)
-helpstr = f.getvalue()
-(options, remaining_args) = parser.parse_args()
-
-
-tempdir = "temp_filesre_split_and_combine/"
-
-def sleep_a_while ():
- time.sleep(0.1)
+import random
+import json
+# use simplejson in place of json for python < 2.6
+#try:
+# import json
+#except ImportError:
+# import simplejson
+# json = simplejson
-if options.verbose:
- verbose_output = sys.stderr
-else:
- verbose_output =open("/dev/null", "w")
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
# Tasks
@@ -178,8 +84,7 @@ else:
#
# split_fasta_file
#
- at posttask(sleep_a_while)
- at posttask(lambda: verbose_output.write("Split into %d files\n" % options.jobs_per_task))
+ at posttask(lambda: verbose_output.write(" Split into %d files\n" % JOBS_PER_TASK))
@files(tempdir + "original.fa", tempdir + "files.split.success")
def split_fasta_file (input_file, success_flag):
#
@@ -191,25 +96,26 @@ def split_fasta_file (input_file, success_flag):
os.unlink(f)
- import random
random.seed()
- for i in range(options.jobs_per_task):
- open(tempdir + "files.split.%03d.fa" % i, "w")
+ for i in range(JOBS_PER_TASK):
+ with open(tempdir + "files.split.%03d.fa" % i, "w") as oo:
+ pass
- open(success_flag, "w")
+ with open(success_flag, "w") as oo:
+ pass
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
#
# align_sequences
#
- at posttask(sleep_a_while)
- at posttask(lambda: verbose_output.write("Sequences aligned\n"))
+ at posttask(lambda: verbose_output.write(" Sequences aligned\n"))
@follows(split_fasta_file)
@files_re(tempdir + "files.split.*.fa", # find all .fa files
".fa$", ".aln") # fa -> aln
def align_sequences (input_file, output_filename):
- open(output_filename, "w").write("%s\n" % output_filename)
+ with open(output_filename, "w") as oo:
+ oo.write("%s\n" % output_filename)
@@ -217,8 +123,7 @@ def align_sequences (input_file, output_filename):
#
# percentage_identity
#
- at posttask(sleep_a_while)
- at posttask(lambda: verbose_output.write("%Identity calculated\n"))
+ at posttask(lambda: verbose_output.write(" %Identity calculated\n"))
@files_re(align_sequences, # find all results from align_sequences
r"(.*\.)(.+).aln$", # match file name root and substitute
r'\g<0>', # the original file
@@ -227,8 +132,10 @@ def align_sequences (input_file, output_filename):
r"\2") # extra parameter to remember the file index
def percentage_identity (input_file, output_files, split_index):
(output_filename, success_flag_filename) = output_files
- open(output_filename, "w").write("%s\n" % split_index)
- open(success_flag_filename, "w")
+ with open(output_filename, "w") as oo:
+ oo.write("%s\n" % split_index)
+ with open(success_flag_filename, "w") as oo:
+ pass
@@ -236,8 +143,7 @@ def percentage_identity (input_file, output_files, split_index):
#
# combine_results
#
- at posttask(lambda: verbose_output.write("Results recombined\n"))
- at posttask(sleep_a_while)
+ at posttask(lambda: verbose_output.write(" Results recombined\n"))
@files_re(percentage_identity, combine(r".*.pcid$"),
[tempdir + "all.combine_results",
tempdir + "all.combine_results_success"])
@@ -248,49 +154,38 @@ def combine_results (input_files, output_files):
(output_filename, success_flag_filename) = output_files
out = open(output_filename, "w")
for inp, flag in input_files:
- out.write(open(inp).read())
- open(success_flag_filename, "w")
+ with open(inp) as ii:
+ out.write(ii.read())
+ out.close()
+ with open(success_flag_filename, "w") as oo:
+ pass
-def start_pipeline_afresh ():
- """
- Recreate directory and starting file
- """
- print("Start again", file=verbose_output)
- import os
- os.system("rm -rf %s" % tempdir)
- os.makedirs(tempdir)
- open(tempdir + "original.fa", "w").close()
- sleep_a_while ()
+
+
+
+import unittest, shutil
+class Test_ruffus(unittest.TestCase):
+
+ def tearDown(self):
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+ def setUp(self):
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+ os.makedirs(tempdir)
+ with open(tempdir + "original.fa", "w") as oo:
+ pass
+
+ def test_ruffus (self):
+ pipeline_run(multiprocess = 100, verbose = 0)
+
if __name__ == '__main__':
- if options.start_again:
- start_pipeline_afresh()
- if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
- verbose = options.verbose,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
-
- elif options.dependency_file:
- pipeline_printout_graph ( open(options.dependency_file, "w"),
- options.dependency_graph_format,
- options.target_tasks,
- options.forced_tasks,
- draw_vertically = not options.draw_horizontally,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- no_key_legend = options.no_key_legend_in_graph)
- elif options.debug:
- start_pipeline_afresh()
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- logger = stderr_logger if options.verbose else black_hole_logger,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- verbose = options.verbose)
- os.system("rm -rf %s" % tempdir)
- print("OK")
- else:
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- logger = stderr_logger if options.verbose else black_hole_logger,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- verbose = options.verbose)
+ unittest.main()
diff --git a/ruffus/test/test_follows_mkdir.py b/ruffus/test/test_follows_mkdir.py
index a7d7739..98be13c 100755
--- a/ruffus/test/test_follows_mkdir.py
+++ b/ruffus/test/test_follows_mkdir.py
@@ -1,145 +1,36 @@
#!/usr/bin/env python
from __future__ import print_function
"""
-
test_follows_mkdir.py
-
- test make directory dependencies
-
- use :
- -j N / --jobs N to specify multitasking
- -v to see the jobs in action
- -n / --just_print to see what jobs would run
-
"""
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# options
-
+import os
+import sys
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
-from optparse import OptionParser
-import sys, os
-import os.path
-try:
- import StringIO as io
-except:
- import io as io
-import re,time
-
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-if __name__ == '__main__':
- module_name = os.path.split(sys.argv[0])[1]
- module_name = os.path.splitext(module_name)[0];
-else:
- module_name = __name__
-
-
-
-import ruffus
-parser = OptionParser(version="%%prog v1.0, ruffus v%s" % ruffus.ruffus_version.__version)
-parser.add_option("-t", "--target_tasks", dest="target_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Target task(s) of pipeline.")
-parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Pipeline task(s) which will be included even if they are up to date.")
-parser.add_option("-j", "--jobs", dest="jobs",
- default=1,
- metavar="jobs",
- type="int",
- help="Specifies the number of jobs (commands) to run simultaneously.")
-parser.add_option("-v", "--verbose", dest = "verbose",
- action="count", default=0,
- help="Print more verbose messages for each additional verbose level.")
-parser.add_option("-d", "--dependency", dest="dependency_file",
- #default="simple.svg",
- metavar="FILE",
- type="string",
- help="Print a dependency graph of the pipeline that would be executed "
- "to FILE, but do not execute it.")
-parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
- metavar="FORMAT",
- type="string",
- default = 'svg',
- help="format of dependency graph file. Can be 'ps' (PostScript), "+
- "'svg' 'svgz' (Structured Vector Graphics), " +
- "'png' 'gif' (bitmap graphics) etc ")
-parser.add_option("-n", "--just_print", dest="just_print",
- action="store_true", default=False,
- help="Print a description of the jobs that would be executed, "
- "but do not execute them.")
-parser.add_option("-M", "--minimal_rebuild_mode", dest="minimal_rebuild_mode",
- action="store_true", default=False,
- help="Rebuild a minimum of tasks necessary for the target. "
- "Ignore upstream out of date tasks if intervening tasks are fine.")
-parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
- action="store_true", default=False,
- help="Do not print out legend and key for dependency graph.")
-parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
- action="store_true", default=False,
- help="Draw horizontal dependency graph.")
-
-parameters = [
- ]
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
-
-
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# imports
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-
-import re
-import operator
-import sys,os
-from collections import defaultdict
-import random
-
-sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-
-# use simplejson in place of json for python < 2.6
try:
- import json
-except ImportError:
- import simplejson
- json = simplejson
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# Main logic
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
-# get help string
-f =io.StringIO()
-parser.print_help(f)
-helpstr = f.getvalue()
-(options, remaining_args) = parser.parse_args()
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -168,36 +59,29 @@ class Test_task_mkdir(unittest.TestCase):
delete directories
"""
for d in 'abcde':
- fullpath = os.path.join(exe_path, d)
+ fullpath = os.path.join(os.path.dirname(__file__), d)
os.rmdir(fullpath)
def test_mkdir (self):
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- logger = stderr_logger if options.verbose else black_hole_logger,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- verbose = options.verbose)
+ pipeline_run(multiprocess = 10, verbose = 0)
for d in 'abcde':
- fullpath = os.path.join(exe_path, d)
+ fullpath = os.path.join(os.path.dirname(__file__), d)
self.assertTrue(os.path.exists(fullpath))
+ def test_newstyle_mkdir (self):
+ test_pipeline = Pipeline("test")
+ test_pipeline.follows(task_which_makes_directories, mkdir(directories), mkdir('c'), mkdir('d', 'e'), mkdir('e'))
+ test_pipeline.run(multiprocess = 10, verbose = 0)
+
+ for d in 'abcde':
+ fullpath = os.path.join(os.path.dirname(__file__), d)
+ self.assertTrue(os.path.exists(fullpath))
+
+
+
if __name__ == '__main__':
- if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
- verbose = options.verbose,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
-
- elif options.dependency_file:
- pipeline_printout_graph ( open(options.dependency_file, "w"),
- options.dependency_graph_format,
- options.target_tasks,
- options.forced_tasks,
- draw_vertically = not options.draw_horizontally,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- no_key_legend = options.no_key_legend_in_graph)
- else:
- sys.argv= sys.argv[0:1]
- unittest.main()
+ unittest.main()
diff --git a/ruffus/test/test_graphviz.py b/ruffus/test/test_graphviz.py
index 34cdc37..036495b 100755
--- a/ruffus/test/test_graphviz.py
+++ b/ruffus/test/test_graphviz.py
@@ -1,66 +1,35 @@
#!/usr/bin/env python
from __future__ import print_function
"""
-
- play_with_colours.py
- [--log_file PATH]
- [--verbose]
+ test_graphviz.py
"""
import unittest
-################################################################################
-#
-# test
-#
-#
-# Copyright (c) 7/13/2010 Leo Goodstadt
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#################################################################################
-
-import sys, os
-
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-if __name__ == '__main__':
- module_name = os.path.split(sys.argv[0])[1]
- module_name = os.path.splitext(module_name)[0];
-else:
- module_name = __name__
-
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+import os
+import sys
-# options
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
-from optparse import OptionParser
try:
- import StringIO as io
-except:
- import io as io
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
+JobSignalledBreak = ruffus.ruffus_exceptions.JobSignalledBreak
@@ -71,8 +40,10 @@ except:
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-from ruffus import *
-from ruffus.ruffus_exceptions import JobSignalledBreak
+try:
+ from StringIO import StringIO
+except:
+ from io import StringIO, BytesIO
@@ -172,11 +143,6 @@ def Downstream_task2_ignored(infile, outfile):
-try:
- from StringIO import StringIO
-except:
- from io import StringIO
-
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -204,26 +170,68 @@ class Test_graphviz(unittest.TestCase):
"""Make sure annotations from graphviz appear in dot
"""
- s = StringIO()
+ if sys.hexversion >= 0x03000000:
+ # everything is unicode in python3
+ s = BytesIO()
+ else:
+ s = StringIO()
+
+
pipeline_printout_graph (
s,
# use flowchart file name extension to decide flowchart format
# e.g. svg, jpg etc.
"dot",
[Final_target, Up_to_date_final_target])
- self.assertTrue('[URL="http://cnn.com", color="#FF0000", fillcolor="#FFCCCC", fontcolor="#4B6000", height=1.5, label=<What is this?<BR/> What <FONT COLOR="red">is</FONT>this???>, pencolor="#FF0000", peripheries=5, shape=component, style=dashed]' in s.getvalue())
-
-
-
+ self.assertTrue('[URL="http://cnn.com", color="#FF0000", fillcolor="#FFCCCC", fontcolor="#4B6000", height=1.5, label=<What is this?<BR/> What <FONT COLOR="red">is</FONT>this???>, pencolor="#FF0000", peripheries=5, shape=component, style=dashed]' in s.getvalue().decode())
+
+
+ def test_newstyle_graphviz_dot(self):
+ test_pipeline = Pipeline("test")
+ test_pipeline.check_if_uptodate (Up_to_date_task1, lambda : (False, ""))
+ test_pipeline.follows(Up_to_date_task2, Up_to_date_task1)\
+ .check_if_uptodate (lambda : (False, ""))\
+ .graphviz(URL='"http://cnn.com"', fillcolor = '"#FFCCCC"',
+ color = '"#FF0000"', pencolor='"#FF0000"', fontcolor='"#4B6000"',
+ label_suffix = "???", label_prefix = "What is this?<BR/> ",
+ label = "<What <FONT COLOR=\"red\">is</FONT>this>",
+ shape= "component", height = 1.5, peripheries = 5,
+ style="dashed")
+ test_pipeline.follows(Up_to_date_task3, Up_to_date_task2)\
+ .check_if_uptodate (lambda : (False, ""))
+ test_pipeline.follows(Up_to_date_final_target, Up_to_date_task3)\
+ .check_if_uptodate (lambda : (False, ""))
+ test_pipeline.follows(Explicitly_specified_task, Up_to_date_task1)\
+ .check_if_uptodate (lambda : (False, ""))
+ test_pipeline.follows(Task_to_run1, Explicitly_specified_task)
+ test_pipeline.follows(Task_to_run2, Task_to_run1)
+ test_pipeline.follows(Task_to_run3, Task_to_run2)
+ test_pipeline.follows(Up_to_date_task_forced_to_rerun, Task_to_run2)\
+ .check_if_uptodate (lambda : (False, ""))
+ test_pipeline.follows(Final_target, Up_to_date_task_forced_to_rerun, Task_to_run3)
+ test_pipeline.follows(Downstream_task1_ignored, Final_target)
+ test_pipeline.follows(Downstream_task2_ignored, Final_target)
+
+ if sys.hexversion >= 0x03000000:
+ # everything is unicode in python3
+ s = BytesIO()
+ else:
+ s = StringIO()
+
+
+ test_pipeline.printout_graph (
+ s,
+ # use flowchart file name extension to decide flowchart format
+ # e.g. svg, jpg etc.
+ "dot",
+ [Final_target, Up_to_date_final_target])
+ self.assertTrue('[URL="http://cnn.com", color="#FF0000", fillcolor="#FFCCCC", fontcolor="#4B6000", height=1.5, label=<What is this?<BR/> What <FONT COLOR="red">is</FONT>this???>, pencolor="#FF0000", peripheries=5, shape=component, style=dashed]' in s.getvalue().decode())
#
# Necessary to protect the "entry point" of the program under windows.
# see: http://docs.python.org/library/multiprocessing.html#multiprocessing-programming
#
if __name__ == '__main__':
- #pipeline_printout(sys.stdout, [test_product_task], verbose = 5)
- #pipeline_printout_graph( "test.png", "png", [Final_target, Up_to_date_final_target])
- #pipeline_printout_graph( "test.dot", "dot", [Final_target, Up_to_date_final_target])
unittest.main()
diff --git a/ruffus/test/test_inputs_with_multiple_args_raising_exception.py b/ruffus/test/test_inputs_with_multiple_args_raising_exception.py
index c1be929..35214ac 100755
--- a/ruffus/test/test_inputs_with_multiple_args_raising_exception.py
+++ b/ruffus/test/test_inputs_with_multiple_args_raising_exception.py
@@ -3,139 +3,58 @@ from __future__ import print_function
"""
test_inputs_with_multiple_args_raising_exception.py
-
- inputs with multiple arguments should raise an exception
-
-"""
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# options
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-from optparse import OptionParser
-import sys, os
-import os.path
-try:
- import StringIO as io
-except:
- import io as io
-import re,time
-
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-if __name__ == '__main__':
- module_name = os.path.split(sys.argv[0])[1]
- module_name = os.path.splitext(module_name)[0];
-else:
- module_name = __name__
-
-
-
-import ruffus
-print("\tRuffus Version = ", ruffus.__version__)
-parser = OptionParser(version="%%prog v1.0, ruffus v%s" % ruffus.ruffus_version.__version)
-parser.add_option("-t", "--target_tasks", dest="target_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Target task(s) of pipeline.")
-parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Pipeline task(s) which will be included even if they are up to date.")
-parser.add_option("-j", "--jobs", dest="jobs",
- default=1,
- metavar="jobs",
- type="int",
- help="Specifies the number of jobs (commands) to run simultaneously.")
-parser.add_option("-v", "--verbose", dest = "verbose",
- action="count", default=0,
- help="Print more verbose messages for each additional verbose level.")
-parser.add_option("-d", "--dependency", dest="dependency_file",
- #default="simple.svg",
- metavar="FILE",
- type="string",
- help="Print a dependency graph of the pipeline that would be executed "
- "to FILE, but do not execute it.")
-parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
- metavar="FORMAT",
- type="string",
- default = 'svg',
- help="format of dependency graph file. Can be 'ps' (PostScript), "+
- "'svg' 'svgz' (Structured Vector Graphics), " +
- "'png' 'gif' (bitmap graphics) etc ")
-parser.add_option("-n", "--just_print", dest="just_print",
- action="store_true", default=False,
- help="Print a description of the jobs that would be executed, "
- "but do not execute them.")
-parser.add_option("-M", "--minimal_rebuild_mode", dest="minimal_rebuild_mode",
- action="store_true", default=False,
- help="Rebuild a minimum of tasks necessary for the target. "
- "Ignore upstream out of date tasks if intervening tasks are fine.")
-parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
- action="store_true", default=False,
- help="Do not print out legend and key for dependency graph.")
-parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
- action="store_true", default=False,
- help="Draw horizontal dependency graph.")
-
-parameters = [
- ]
-
+ inputs with multiple arguments should raise an exception
+"""
+import os
+import sys
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
-# imports
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+try:
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
-import re
-import operator
-import sys,os
-from collections import defaultdict
-import random
-sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-# use simplejson in place of json for python < 2.6
-try:
- import json
-except ImportError:
- import simplejson
- json = simplejson
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-# Main logic
+# imports
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+import unittest
+import json
+## use simplejson in place of json for python < 2.6
+#try:
+# import json
+#except ImportError:
+# import simplejson
+# json = simplejson
-# get help string
-f =io.StringIO()
-parser.print_help(f)
-helpstr = f.getvalue()
-(options, remaining_args) = parser.parse_args()
-
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -149,12 +68,14 @@ try:
for f in o:
open(f, 'w')
except ruffus.ruffus_exceptions.error_task_transform_inputs_multiple_args:
- print("\tExpected exception thrown")
- sys.exit(0)
+ print("\tExpected exception thrown 1")
+except ruffus.ruffus_exceptions.error_inputs_multiple_args:
+ print("\tExpected exception thrown 2")
-raise Exception("Inputs(...) with multiple arguments should have thrown an exception")
+def task_2 (i, o):
+ for f in o:
+ open(f, 'w')
-import unittest
class Test_task_mkdir(unittest.TestCase):
@@ -162,7 +83,7 @@ class Test_task_mkdir(unittest.TestCase):
"""
"""
pass
-
+
def tearDown (self):
"""
"""
@@ -170,27 +91,32 @@ class Test_task_mkdir(unittest.TestCase):
def test_no_re_match (self):
- pipeline_run([task_1], options.forced_tasks, multiprocess = options.jobs,
- logger = stderr_logger if options.verbose else black_hole_logger,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- verbose = options.verbose)
-
+ try:
+ pipeline_run(multiprocess = 10, verbose = 0)
+ except:
+ return
+ raise Exception("Inputs(...) with multiple arguments should have thrown an exception")
+
+ def test_newstyle_no_re_match (self):
+ try:
+ test_pipeline = Pipeline("test")
+ test_pipeline.transform(task_func = task_2,
+ input = None,
+ filter = regex("b"),
+ replace_inputs = inputs("a", "b"),
+ output = "task_1.output")
+ test_pipeline.run(multiprocess = 10, verbose = 0)
+ except ruffus.ruffus_exceptions.error_task_transform_inputs_multiple_args:
+ print("\tExpected exception thrown 1")
+ return
+ except ruffus.ruffus_exceptions.error_inputs_multiple_args:
+ print("\tExpected exception thrown 2")
+ return
+ raise Exception("Inputs(...) with multiple arguments should have thrown an exception")
+
+
+
if __name__ == '__main__':
- if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
- verbose = options.verbose,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
-
- elif options.dependency_file:
- pipeline_printout_graph ( open(options.dependency_file, "w"),
- options.dependency_graph_format,
- options.target_tasks,
- options.forced_tasks,
- draw_vertically = not options.draw_horizontally,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- no_key_legend = options.no_key_legend_in_graph)
- else:
- sys.argv= sys.argv[0:1]
- unittest.main()
+ unittest.main()
diff --git a/ruffus/test/test_job_completion_checksums.py b/ruffus/test/test_job_completion_checksums.py
index caea11d..ca96b84 100755
--- a/ruffus/test/test_job_completion_checksums.py
+++ b/ruffus/test/test_job_completion_checksums.py
@@ -8,32 +8,51 @@ from __future__ import print_function
"""
-
-import unittest
import os
import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+for attr in "pipeline_run", "pipeline_printout", "suffix", "transform", "split", "merge", "dbdict", "Pipeline":
+ globals()[attr] = getattr (ruffus, attr)
+get_default_history_file_name = ruffus.task.get_default_history_file_name
+RUFFUS_HISTORY_FILE = ruffus.ruffus_utility.RUFFUS_HISTORY_FILE
+CHECKSUM_FILE_TIMESTAMPS = ruffus.ruffus_utility.CHECKSUM_FILE_TIMESTAMPS
+CHECKSUM_HISTORY_TIMESTAMPS = ruffus.ruffus_utility.CHECKSUM_HISTORY_TIMESTAMPS
+CHECKSUM_FUNCTIONS = ruffus.ruffus_utility.CHECKSUM_FUNCTIONS
+CHECKSUM_FUNCTIONS_AND_PARAMS = ruffus.ruffus_utility.CHECKSUM_FUNCTIONS_AND_PARAMS
+RethrownJobError = ruffus.ruffus_exceptions.RethrownJobError
+
+
+
+
+#___________________________________________________________________________
+#
+# imports
+#___________________________________________________________________________
+
+
+import unittest
import shutil
try:
from StringIO import StringIO
except:
from io import StringIO
import time
+import re
-import sys, re
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0, os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import (pipeline_run, pipeline_printout, suffix, transform, split,
- merge, dbdict)
-from ruffus.task import get_default_history_file_name
-from ruffus.ruffus_utility import (RUFFUS_HISTORY_FILE,
- CHECKSUM_FILE_TIMESTAMPS,
- CHECKSUM_HISTORY_TIMESTAMPS,
- CHECKSUM_FUNCTIONS,
- CHECKSUM_FUNCTIONS_AND_PARAMS)
-from ruffus.ruffus_exceptions import RethrownJobError
-
possible_chksms = list(range(CHECKSUM_FUNCTIONS_AND_PARAMS + 1))
workdir = 'tmp_test_job_completion/'
input_file = os.path.join(workdir, 'input.txt')
@@ -56,7 +75,7 @@ def transform_raise_error(in_name, out_name, how_many):
with open(out_name, 'w') as outfile:
with open(in_name) as ii:
outfile.write(ii.read())
- if 'okay' not in runtime_data:
+ if 'okay' not in how_many:
raise RuntimeError("'okay' wasn't in runtime_data!")
@split(input_file, split1_outputs)
@@ -83,6 +102,7 @@ def merge2(in_names, out_name):
def cleanup_tmpdir():
os.system('rm -f %s %s' % (os.path.join(workdir, '*'), get_default_history_file_name()))
+count_pipelines = 0
class TestJobCompletion(unittest.TestCase):
def setUp(self):
@@ -91,7 +111,38 @@ class TestJobCompletion(unittest.TestCase):
except OSError:
pass
- def test_ouput_doesnt_exist(self):
+
+ def create_pipeline (self):
+ """
+ Create new pipeline on the fly without using decorators
+ """
+ global count_pipelines
+ count_pipelines = count_pipelines + 1
+ test_pipeline = Pipeline("test %d" % count_pipelines)
+
+ test_pipeline.transform(task_func = transform1,
+ input = input_file,
+ filter = suffix('.txt'),
+ output = '.output',
+ extras = [runtime_data])
+
+ test_pipeline.transform(task_func = transform_raise_error,
+ input = input_file,
+ filter = suffix('.txt'),
+ output = '.output',
+ extras = [runtime_data])
+
+ test_pipeline.split( task_func = split1,
+ input = input_file,
+ output = split1_outputs)
+
+ test_pipeline.merge( task_func = merge2,
+ input = split1,
+ output = merge2_output)
+ return test_pipeline
+
+
+ def test_output_doesnt_exist(self):
"""Input file exists, output doesn't exist"""
# output doesn't exist-- should run for all levels
# create a new input file
@@ -102,12 +153,12 @@ class TestJobCompletion(unittest.TestCase):
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [transform1], verbose=6, checksum_level=chksm)
- self.assertTrue(re.search(r'Job needs update: Missing file\n\s+\[tmp_test_job_completion/input.output\]'
- , s.getvalue()))
+ self.assertTrue(re.search(r'Job needs update:.*Missing file.*\[tmp_test_job_completion/input.output\]'
+ , s.getvalue(), re.DOTALL))
- def test_ouput_out_of_date(self):
+ def test_output_out_of_date(self):
"""Input file exists, output out of date"""
# output exists but is out of date-- should run for all levels
cleanup_tmpdir()
@@ -125,9 +176,9 @@ class TestJobCompletion(unittest.TestCase):
self.assertIn('Input files:', s.getvalue())
self.assertIn('Output files:', s.getvalue())
else:
- self.assertIn('Previous incomplete run leftover', s.getvalue())
+ self.assertIn('left over from a failed run?', s.getvalue())
- def test_ouput_timestamp_okay(self):
+ def test_output_timestamp_okay(self):
"""Input file exists, output timestamp up to date"""
# output exists and timestamp is up to date-- not run for lvl 0, run for all others
cleanup_tmpdir()
@@ -145,10 +196,10 @@ class TestJobCompletion(unittest.TestCase):
pass
else:
self.assertIn('Job needs update:', s.getvalue())
- self.assertIn('Previous incomplete run leftover',
+ self.assertIn('left over from a failed run?',
s.getvalue())
- def test_ouput_up_to_date(self):
+ def test_output_up_to_date(self):
"""Input file exists, output up to date"""
# output is up to date-- not run for any levels
cleanup_tmpdir()
@@ -162,7 +213,7 @@ class TestJobCompletion(unittest.TestCase):
#self.assertIn('Job up-to-date', s.getvalue())
pass
- def test_ouput_up_to_date_func_changed(self):
+ def test_output_up_to_date_func_changed(self):
"""Input file exists, output up to date, function body changed"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
@@ -185,7 +236,7 @@ class TestJobCompletion(unittest.TestCase):
#self.assertIn('Job up-to-date', s.getvalue())
pass
- def test_ouput_up_to_date_func_changed(self):
+ def test_output_up_to_date_func_changed(self):
"""Input file exists, output up to date, function body changed"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
@@ -215,7 +266,7 @@ class TestJobCompletion(unittest.TestCase):
split1.func_code, transform1.func_code = transform1.func_code, split1.func_code
- def test_ouput_up_to_date_param_changed(self):
+ def test_output_up_to_date_param_changed(self):
"""Input file exists, output up to date, parameter to function changed"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
@@ -235,6 +286,243 @@ class TestJobCompletion(unittest.TestCase):
#self.assertIn('Job up-to-date', s.getvalue())
pass
+ def test_split_output(self):
+ """test multiple-output checksums"""
+ # outputs out of date
+ cleanup_tmpdir()
+ with open(input_file, 'w') as outfile:
+ outfile.write('testme')
+ pipeline_run([split1], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
+ time.sleep(.5)
+ with open(input_file, 'w') as outfile:
+ outfile.write('testme')
+
+ for chksm in possible_chksms:
+ s = StringIO()
+ pipeline_printout(s, [split1], verbose=6, checksum_level=chksm)
+ self.assertIn('Job needs update:', s.getvalue())
+
+ # all outputs incorrectly generated
+ cleanup_tmpdir()
+ with open(input_file, 'w') as outfile:
+ outfile.write('testme')
+ time.sleep(.5)
+ for f in split1_outputs:
+ with open(f, 'w') as outfile:
+ outfile.write('testme')
+ for chksm in possible_chksms:
+ s = StringIO()
+ pipeline_printout(s, [split1], verbose=6, checksum_level=chksm)
+ if chksm >= CHECKSUM_HISTORY_TIMESTAMPS:
+ self.assertIn('Job needs update:', s.getvalue())
+ self.assertIn('left over from a failed run?',
+ s.getvalue())
+ else:
+ #self.assertIn('Job up-to-date', s.getvalue())
+ pass
+
+ # one output incorrectly generated
+ cleanup_tmpdir()
+ with open(input_file, 'w') as outfile:
+ outfile.write('testme')
+ pipeline_run([split1], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
+ job_history = dbdict.open(get_default_history_file_name(), picklevalues=True)
+ del job_history[os.path.relpath(split1_outputs[0])]
+
+ for chksm in possible_chksms:
+ s = StringIO()
+ pipeline_printout(s, [split1], verbose=6, checksum_level=chksm)
+ if chksm >= CHECKSUM_HISTORY_TIMESTAMPS:
+ self.assertIn('Job needs update:', s.getvalue())
+ self.assertIn('left over from a failed run?',
+ s.getvalue())
+ else:
+ #self.assertIn('Job up-to-date', s.getvalue())
+ pass
+
+ def test_merge_output(self):
+ """test multiple-input checksums"""
+ # one output incorrectly generated
+ cleanup_tmpdir()
+ with open(input_file, 'w') as outfile:
+ outfile.write('testme')
+ pipeline_run([split1], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
+ job_history = dbdict.open(get_default_history_file_name(), picklevalues=True)
+ del job_history[os.path.relpath(split1_outputs[0])]
+
+ for chksm in possible_chksms:
+ s = StringIO()
+ pipeline_printout(s, [merge2], verbose=6, checksum_level=chksm)
+ if chksm >= CHECKSUM_HISTORY_TIMESTAMPS:
+ self.assertIn('Job needs update:', s.getvalue())
+ self.assertIn('left over from a failed run?', s.getvalue())
+ else:
+ #self.assertIn('Job up-to-date', s.getvalue())
+ pass
+
+ # make sure the jobs run fine
+ cleanup_tmpdir()
+ with open(input_file, 'w') as outfile:
+ outfile.write('testme')
+ pipeline_run([merge2], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
+ for chksm in possible_chksms:
+ s = StringIO()
+ pipeline_printout(s, [merge2], verbose=6, checksum_level=chksm)
+ #self.assertIn('Job up-to-date', s.getvalue())
+ self.assertNotIn('Job needs update:', s.getvalue())
+ self.assertNotIn('left over from a failed run?', s.getvalue())
+
+
+
+
+ def test_newstyle_output_doesnt_exist(self):
+ """Input file exists, output doesn't exist"""
+ # output doesn't exist-- should run for all levels
+ # create a new input file
+ cleanup_tmpdir()
+ with open(input_file, 'w') as outfile:
+ outfile.write('testme')
+
+ for chksm in possible_chksms:
+ s = StringIO()
+ self.create_pipeline().printout(s, [transform1], verbose=6, checksum_level=chksm)
+ self.assertTrue(re.search(r'Job needs update:.*Missing file.*\[tmp_test_job_completion/input.output\]'
+ , s.getvalue(), re.DOTALL))
+
+
+
+ def test_newstyle_output_out_of_date(self):
+ """Input file exists, output out of date"""
+ # output exists but is out of date-- should run for all levels
+ cleanup_tmpdir()
+ with open(transform1_out, 'w') as outfile:
+ outfile.write('testme')
+ time.sleep(0.1)
+ with open(input_file, 'w') as outfile:
+ outfile.write('testme')
+
+ for chksm in possible_chksms:
+ s = StringIO()
+ self.create_pipeline().printout(s, [transform1], verbose=6, checksum_level=chksm)
+ self.assertIn('Job needs update:', s.getvalue())
+ if chksm == CHECKSUM_FILE_TIMESTAMPS:
+ self.assertIn('Input files:', s.getvalue())
+ self.assertIn('Output files:', s.getvalue())
+ else:
+ self.assertIn('left over from a failed run?', s.getvalue())
+
+ def test_newstyle_output_timestamp_okay(self):
+ """Input file exists, output timestamp up to date"""
+ # output exists and timestamp is up to date-- not run for lvl 0, run for all others
+ cleanup_tmpdir()
+ with open(input_file, 'w') as outfile:
+ outfile.write('testme')
+ time.sleep(0.1)
+ with open(transform1_out, 'w') as outfile:
+ outfile.write('testme')
+
+ for chksm in possible_chksms:
+ s = StringIO()
+ self.create_pipeline().printout(s, [transform1], verbose=6, checksum_level=chksm)
+ if chksm == CHECKSUM_FILE_TIMESTAMPS:
+ #self.assertIn('Job up-to-date', s.getvalue())
+ pass
+ else:
+ self.assertIn('Job needs update:', s.getvalue())
+ self.assertIn('left over from a failed run?',
+ s.getvalue())
+
+ def test_newstyle_output_up_to_date(self):
+ """Input file exists, output up to date"""
+ test_pipeline = self.create_pipeline()
+ # output is up to date-- not run for any levels
+ cleanup_tmpdir()
+ with open(input_file, 'w') as outfile:
+ outfile.write('testme')
+ test_pipeline.run([transform1], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
+
+ for chksm in possible_chksms:
+ s = StringIO()
+ test_pipeline.printout(s, [transform1], verbose=6, checksum_level=chksm)
+ #self.assertIn('Job up-to-date', s.getvalue())
+ pass
+
+ def test_newstyle_output_up_to_date_func_changed(self):
+ """Input file exists, output up to date, function body changed"""
+ test_pipeline = self.create_pipeline()
+ # output is up to date, but function body changed (e.g., source different)
+ cleanup_tmpdir()
+ with open(input_file, 'w') as outfile:
+ outfile.write('testme')
+ test_pipeline.run([transform1], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
+ if sys.hexversion >= 0x03000000:
+ transform1.__code__ = split1.__code__ # simulate source change
+ else:
+ transform1.func_code = split1.func_code # simulate source change
+
+ for chksm in possible_chksms:
+ s = StringIO()
+ test_pipeline.printout(s, [transform1], verbose=6, checksum_level=chksm)
+ if chksm >= CHECKSUM_FUNCTIONS:
+ self.assertIn('Job needs update:', s.getvalue())
+ self.assertIn('Pipeline function has changed',
+ s.getvalue())
+ else:
+ #self.assertIn('Job up-to-date', s.getvalue())
+ pass
+
+ def test_newstyle_output_up_to_date_func_changed(self):
+ """Input file exists, output up to date, function body changed"""
+ test_pipeline = self.create_pipeline()
+ # output is up to date, but function body changed (e.g., source different)
+ cleanup_tmpdir()
+ with open(input_file, 'w') as outfile:
+ outfile.write('testme')
+ test_pipeline.run([transform1], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
+ # simulate source change
+ if sys.hexversion >= 0x03000000:
+ split1.__code__, transform1.__code__ = transform1.__code__, split1.__code__
+ else:
+ split1.func_code, transform1.func_code = transform1.func_code, split1.func_code
+
+ for chksm in possible_chksms:
+ s = StringIO()
+ test_pipeline.printout(s, [transform1], verbose=6, checksum_level=chksm)
+ if chksm >= CHECKSUM_FUNCTIONS:
+ self.assertIn('Job needs update:', s.getvalue())
+ self.assertIn('Pipeline function has changed',
+ s.getvalue())
+ else:
+ #self.assertIn('Job up-to-date', s.getvalue())
+ pass
+ # clean up our function-changing mess!
+ if sys.hexversion >= 0x03000000:
+ split1.__code__, transform1.__code__ = transform1.__code__, split1.__code__
+ else:
+ split1.func_code, transform1.func_code = transform1.func_code, split1.func_code
+
+
+ def test_newstyle_output_up_to_date_param_changed(self):
+ """Input file exists, output up to date, parameter to function changed"""
+ test_pipeline = self.create_pipeline()
+ # output is up to date, but function body changed (e.g., source different)
+ cleanup_tmpdir()
+ with open(input_file, 'w') as outfile:
+ outfile.write('testme')
+ test_pipeline.run([transform1], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
+ runtime_data.append('different') # simulate change to config file
+
+ for chksm in possible_chksms:
+ s = StringIO()
+ test_pipeline.printout(s, [transform1], verbose=6, checksum_level=chksm)
+ if chksm >= CHECKSUM_FUNCTIONS_AND_PARAMS:
+ self.assertIn('Job needs update:', s.getvalue())
+ self.assertIn('Pipeline parameters have changed',
+ s.getvalue())
+ else:
+ #self.assertIn('Job up-to-date', s.getvalue())
+ pass
+
def test_raises_error(self):
"""run a function that fails but creates output, then check what should run"""
# output is up to date, but function body changed (e.g., source different)
@@ -251,27 +539,50 @@ class TestJobCompletion(unittest.TestCase):
pipeline_printout(s, [transform_raise_error], verbose=6, checksum_level=chksm)
if chksm >= CHECKSUM_HISTORY_TIMESTAMPS:
self.assertIn('Job needs update:', s.getvalue())
- self.assertIn('Previous incomplete run leftover',
+ self.assertIn('left over from a failed run?',
+ s.getvalue())
+ else:
+ #self.assertIn('Job up-to-date', s.getvalue())
+ pass
+
+ def test_newstyle_raises_error(self):
+ """run a function that fails but creates output, then check what should run"""
+ test_pipeline = self.create_pipeline()
+ # output is up to date, but function body changed (e.g., source different)
+ cleanup_tmpdir()
+ with open(input_file, 'w') as outfile:
+ outfile.write('testme')
+ time.sleep(.5)
+ del runtime_data[:]
+ with self.assertRaises(RethrownJobError): # poo. Shouldn't this be RuntimeError?
+ test_pipeline.run([transform_raise_error], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS) # generates output then fails
+ for chksm in possible_chksms:
+ s = StringIO()
+ test_pipeline.printout(s, [transform_raise_error], verbose=6, checksum_level=chksm)
+ if chksm >= CHECKSUM_HISTORY_TIMESTAMPS:
+ self.assertIn('Job needs update:', s.getvalue())
+ self.assertIn('left over from a failed run?',
s.getvalue())
else:
#self.assertIn('Job up-to-date', s.getvalue())
pass
- def test_split_output(self):
+ def test_newstyle_split_output(self):
"""test multiple-output checksums"""
+ test_pipeline = self.create_pipeline()
# outputs out of date
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
- pipeline_run([split1], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
+ test_pipeline.run([split1], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
time.sleep(.5)
with open(input_file, 'w') as outfile:
outfile.write('testme')
for chksm in possible_chksms:
s = StringIO()
- pipeline_printout(s, [split1], verbose=6, checksum_level=chksm)
+ test_pipeline.printout(s, [split1], verbose=6, checksum_level=chksm)
self.assertIn('Job needs update:', s.getvalue())
# all outputs incorrectly generated
@@ -284,10 +595,10 @@ class TestJobCompletion(unittest.TestCase):
outfile.write('testme')
for chksm in possible_chksms:
s = StringIO()
- pipeline_printout(s, [split1], verbose=6, checksum_level=chksm)
+ test_pipeline.printout(s, [split1], verbose=6, checksum_level=chksm)
if chksm >= CHECKSUM_HISTORY_TIMESTAMPS:
self.assertIn('Job needs update:', s.getvalue())
- self.assertIn('Previous incomplete run leftover',
+ self.assertIn('left over from a failed run?',
s.getvalue())
else:
#self.assertIn('Job up-to-date', s.getvalue())
@@ -297,37 +608,38 @@ class TestJobCompletion(unittest.TestCase):
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
- pipeline_run([split1], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
+ test_pipeline.run([split1], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
job_history = dbdict.open(get_default_history_file_name(), picklevalues=True)
del job_history[os.path.relpath(split1_outputs[0])]
for chksm in possible_chksms:
s = StringIO()
- pipeline_printout(s, [split1], verbose=6, checksum_level=chksm)
+ test_pipeline.printout(s, [split1], verbose=6, checksum_level=chksm)
if chksm >= CHECKSUM_HISTORY_TIMESTAMPS:
self.assertIn('Job needs update:', s.getvalue())
- self.assertIn('Previous incomplete run leftover',
+ self.assertIn('left over from a failed run?',
s.getvalue())
else:
#self.assertIn('Job up-to-date', s.getvalue())
pass
- def test_merge_output(self):
+ def test_newstyle_merge_output(self):
"""test multiple-input checksums"""
+ test_pipeline = self.create_pipeline()
# one output incorrectly generated
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
- pipeline_run([split1], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
+ test_pipeline.run([split1], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
job_history = dbdict.open(get_default_history_file_name(), picklevalues=True)
del job_history[os.path.relpath(split1_outputs[0])]
for chksm in possible_chksms:
s = StringIO()
- pipeline_printout(s, [merge2], verbose=6, checksum_level=chksm)
+ test_pipeline.printout(s, [merge2], verbose=6, checksum_level=chksm)
if chksm >= CHECKSUM_HISTORY_TIMESTAMPS:
self.assertIn('Job needs update:', s.getvalue())
- self.assertIn('Previous incomplete run leftover', s.getvalue())
+ self.assertIn('left over from a failed run?', s.getvalue())
else:
#self.assertIn('Job up-to-date', s.getvalue())
pass
@@ -336,19 +648,21 @@ class TestJobCompletion(unittest.TestCase):
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
- pipeline_run([merge2], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
+ test_pipeline.run([merge2], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
for chksm in possible_chksms:
s = StringIO()
- pipeline_printout(s, [merge2], verbose=6, checksum_level=chksm)
+ test_pipeline.printout(s, [merge2], verbose=6, checksum_level=chksm)
#self.assertIn('Job up-to-date', s.getvalue())
self.assertNotIn('Job needs update:', s.getvalue())
- self.assertNotIn('Previous incomplete run leftover', s.getvalue())
-
+ self.assertNotIn('left over from a failed run?', s.getvalue())
def tearDown(self):
shutil.rmtree(workdir)
+ pass
+
+if __name__ == '__main__':
+ unittest.main()
-#if __name__ == '__main__':
# try:
# os.mkdir(workdir)
# except OSError:
diff --git a/ruffus/test/test_job_history_with_exceptions.py b/ruffus/test/test_job_history_with_exceptions.py
index 81329d3..f7b358f 100755
--- a/ruffus/test/test_job_history_with_exceptions.py
+++ b/ruffus/test/test_job_history_with_exceptions.py
@@ -8,33 +8,52 @@ from __future__ import print_function
"""
+workdir = 'tmp_test_job_history_with_exceptions'
+#sub-1s resolution in system?
+one_second_per_job = None
+throw_exception = False
-import unittest
import os
import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+for attr in "pipeline_run", "pipeline_printout", "suffix", "transform", "split", "merge", "dbdict", "follows", "originate", "collate", "formatter", "Pipeline":
+ globals()[attr] = getattr (ruffus, attr)
+RethrownJobError = ruffus.ruffus_exceptions.RethrownJobError
+RUFFUS_HISTORY_FILE = ruffus.ruffus_utility.RUFFUS_HISTORY_FILE
+CHECKSUM_FILE_TIMESTAMPS = ruffus.ruffus_utility.CHECKSUM_FILE_TIMESTAMPS
+get_default_history_file_name = ruffus.ruffus_utility.get_default_history_file_name
+
+
+
+
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+#
+# imports
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+import unittest
import shutil
try:
from StringIO import StringIO
except:
from io import StringIO
-import time
import re
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0, os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-from ruffus import (pipeline_run, pipeline_printout, suffix, transform, split,
- merge, dbdict, follows)
-#from ruffus.combinatorics import *
-from ruffus.ruffus_exceptions import RethrownJobError
-from ruffus.ruffus_utility import (RUFFUS_HISTORY_FILE,
- CHECKSUM_FILE_TIMESTAMPS,
- get_default_history_file_name)
-workdir = 'tmp_test_job_history_with_exceptions'
-#sub-1s resolution in system?
-one_second_per_job = None
-throw_exception = False
#___________________________________________________________________________
#
# generate_initial_files1
@@ -125,6 +144,7 @@ def cleanup_tmpdir():
VERBOSITY = 5
VERBOSITY = 11
+cnt_pipelines = 0
class Test_job_history_with_exceptions(unittest.TestCase):
def setUp(self):
try:
@@ -142,6 +162,45 @@ class Test_job_history_with_exceptions(unittest.TestCase):
pipeline_printout(s, [test_task4], verbose=VERBOSITY, wrap_width = 10000)
#print s.getvalue()
+
+ def create_pipeline (self):
+ #each pipeline has a different name
+ global cnt_pipelines
+ cnt_pipelines = cnt_pipelines + 1
+ test_pipeline = Pipeline("test %d" % cnt_pipelines)
+
+ test_pipeline.originate(task_func = generate_initial_files1,
+ output = [workdir + "/" + prefix + "_name.tmp1" for prefix in "abcd"])
+
+ test_pipeline.originate(task_func = generate_initial_files2,
+ output = [workdir + "/e_name.tmp1", workdir + "/f_name.tmp1"])
+
+ test_pipeline.originate(task_func = generate_initial_files3,
+ output = [workdir + "/g_name.tmp1", workdir + "/h_name.tmp1"])
+
+ test_pipeline.originate(task_func = generate_initial_files4,
+ output = workdir + "/i_name.tmp1")
+
+ test_pipeline.collate( task_func = test_task2,
+ input = [generate_initial_files1,
+ generate_initial_files2,
+ generate_initial_files3,
+ generate_initial_files4],
+ filter = formatter(),
+ output = "{path[0]}/all.tmp2")
+
+ test_pipeline.transform(task_func = test_task3,
+ input = test_task2,
+ filter = suffix(".tmp2"),
+ output = ".tmp3")
+
+ test_pipeline.transform(task_func = test_task4,
+ input = test_task3,
+ filter = suffix(".tmp3"),
+ output = ".tmp4")
+ return test_pipeline
+
+
def test_job_history_with_exceptions_run(self):
"""Run"""
for i in range(1):
@@ -167,6 +226,82 @@ class Test_job_history_with_exceptions(unittest.TestCase):
+ def test_newstyle_recreate_job_history(self):
+ """Run"""
+ test_pipeline = self.create_pipeline()
+ global throw_exception
+ throw_exception = None
+ cleanup_tmpdir()
+
+ #
+ # print "Initial run without creating sqlite file"
+ #
+ test_pipeline.run([test_task4], verbose = 0,
+ checksum_level = CHECKSUM_FILE_TIMESTAMPS,
+ multithread = 10,
+ one_second_per_job = one_second_per_job)
+
+ #
+ # print "printout without sqlite"
+ #
+ s = StringIO()
+ test_pipeline.printout(s, [test_task4], checksum_level = CHECKSUM_FILE_TIMESTAMPS)
+ self.assertTrue(not re.search('Tasks which will be run:.*\n(.*\n)*Task = ', s.getvalue()))
+ #
+ # print "printout expecting sqlite file"
+ #
+ s = StringIO()
+ test_pipeline.printout(s, [test_task4])
+ self.assertTrue(re.search('Tasks which will be run:.*\n(.*\n)*Task = ', s.getvalue()))
+ #
+ # print "Regenerate sqlite file"
+ #
+ test_pipeline.run([test_task4],
+ checksum_level = CHECKSUM_FILE_TIMESTAMPS,
+ history_file = get_default_history_file_name (),
+ multithread = 1,
+ verbose = 0,
+ touch_files_only = 2,
+ one_second_per_job = one_second_per_job)
+ #
+ # print "printout expecting sqlite file"
+ #
+ s = StringIO()
+ test_pipeline.printout(s, [test_task4], verbose = VERBOSITY)
+ succeed = not re.search('Tasks which will be run:.*\n(.*\n)*Task = ', s.getvalue())
+ if not succeed:
+ print(s.getvalue(), file=sys.stderr)
+ self.assertTrue(succeed)
+
+ throw_exception = False
+
+ #
+ def test_newstyle_job_history_with_exceptions_run(self):
+ """Run"""
+ test_pipeline = self.create_pipeline()
+ for i in range(1):
+ cleanup_tmpdir()
+ try:
+ test_pipeline.run([test_task4], verbose = 0,
+ #multithread = 2,
+ one_second_per_job = one_second_per_job)
+ except:
+ pass
+ s = StringIO()
+ test_pipeline.printout(s, [test_task4], verbose=VERBOSITY, wrap_width = 10000)
+ #
+ # task 2 should be up to date because exception was throw in task 3
+ #
+ pipeline_printout_str = s.getvalue()
+ correct_order = not re.search('Tasks which will be run:.*\n(.*\n)*Task = test_task2', pipeline_printout_str)
+ if not correct_order:
+ print(pipeline_printout_str)
+ self.assertTrue(correct_order)
+ sys.stderr.write(".")
+ print()
+
+
+
def test_recreate_job_history(self):
"""Run"""
global throw_exception
@@ -198,7 +333,7 @@ class Test_job_history_with_exceptions(unittest.TestCase):
#
pipeline_run([test_task4],
checksum_level = CHECKSUM_FILE_TIMESTAMPS,
- history_file = ruffus_utility.get_default_history_file_name (),
+ history_file = get_default_history_file_name (),
multithread = 1,
verbose = 0,
touch_files_only = 2,
@@ -215,7 +350,6 @@ class Test_job_history_with_exceptions(unittest.TestCase):
throw_exception = False
-
#___________________________________________________________________________
#
# cleanup
diff --git a/ruffus/test/test_mkdir.py b/ruffus/test/test_mkdir.py
index b58d3bf..204bdbb 100755
--- a/ruffus/test/test_mkdir.py
+++ b/ruffus/test/test_mkdir.py
@@ -3,15 +3,42 @@ from __future__ import print_function
"""
test_mkdir.py
-
test product, combine, permute, combine_with_replacement
"""
+workdir = 'tmp_test_mkdir'
+
-import unittest
import os
import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+
+for attr in "pipeline_run", "pipeline_printout", "transform", "split", "mkdir", "formatter", "Pipeline":
+ globals()[attr] = getattr (ruffus, attr)
+RethrownJobError = ruffus.ruffus_exceptions.RethrownJobError
+RUFFUS_HISTORY_FILE = ruffus.ruffus_utility.RUFFUS_HISTORY_FILE
+CHECKSUM_FILE_TIMESTAMPS = ruffus.ruffus_utility.CHECKSUM_FILE_TIMESTAMPS
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+#
+# imports
+#
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+import unittest
import shutil
try:
from StringIO import StringIO
@@ -19,16 +46,7 @@ except:
from io import StringIO
import time
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0, os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-from ruffus import (pipeline_run, pipeline_printout, suffix, transform, split,
- merge, dbdict, follows)
-from ruffus.ruffus_exceptions import RethrownJobError
-from ruffus.ruffus_utility import (RUFFUS_HISTORY_FILE,
- CHECKSUM_FILE_TIMESTAMPS)
-workdir = 'tmp_test_mkdir'
#sub-1s resolution in system?
#___________________________________________________________________________
#
@@ -59,7 +77,7 @@ def test_transform( infiles, outfile):
@mkdir(generate_initial_files1, formatter(),
"{path[0]}/{basename[0]}.dir2")
def test_transform2():
- print("Loose cannon!", file=sys.stderr)
+ print(" Loose cannon!", file=sys.stderr)
@@ -97,6 +115,30 @@ class Testmkdir(unittest.TestCase):
pipeline_run([test_transform, test_transform2], verbose=0, multiprocess = 2)
+ def test_newstyle_mkdir_run(self):
+ test_pipeline = Pipeline("test")
+
+ test_pipeline.split(task_func = generate_initial_files1,
+ input = 1,
+ output = [workdir + "/" + prefix + "_name.tmp1" for prefix in "abcd"])
+
+ test_pipeline.transform( task_func = test_transform,
+ input = generate_initial_files1,
+ filter = formatter(),
+ output = "{path[0]}/{basename[0]}.dir/{basename[0]}.tmp2")\
+ .mkdir(workdir + "/test1")\
+ .mkdir(workdir + "/test2")\
+ .mkdir(generate_initial_files1, formatter(),
+ ["{path[0]}/{basename[0]}.dir", 3, "{path[0]}/{basename[0]}.dir2"])
+
+ test_pipeline.mkdir(test_transform2, workdir + "/test3")\
+ .mkdir(generate_initial_files1, formatter(),
+ "{path[0]}/{basename[0]}.dir2")
+ cleanup_tmpdir()
+ pipeline_run([test_transform, test_transform2], verbose=0, multiprocess = 2)
+
+
+
#___________________________________________________________________________
#
diff --git a/ruffus/test/test_combinatorics.py b/ruffus/test/test_newstyle_combinatorics.py
similarity index 56%
copy from ruffus/test/test_combinatorics.py
copy to ruffus/test/test_newstyle_combinatorics.py
index a4b5d04..c09ab31 100755
--- a/ruffus/test/test_combinatorics.py
+++ b/ruffus/test/test_newstyle_combinatorics.py
@@ -9,34 +9,52 @@ from __future__ import print_function
"""
-import unittest
+workdir = 'tmp_test_combinatorics'
+#sub-1s resolution in system?
+one_second_per_job = None
+
import os
import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+try:
+ attrlist = ruffus.combinatorics.__all__
+except AttributeError:
+ attrlist = dir (ruffus.combinatorics)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus.combinatorics, attr)
+
+for attr in "pipeline_run", "pipeline_printout", "suffix", "transform", "split", "merge", "dbdict", "follows", "Pipeline", "formatter", "output_from":
+ globals()[attr] = getattr (ruffus, attr)
+RethrownJobError = ruffus.ruffus_exceptions.RethrownJobError
+RUFFUS_HISTORY_FILE = ruffus.ruffus_utility.RUFFUS_HISTORY_FILE
+CHECKSUM_FILE_TIMESTAMPS = ruffus.ruffus_utility.CHECKSUM_FILE_TIMESTAMPS
+
+
+import unittest
import shutil
try:
from StringIO import StringIO
except:
from io import StringIO
-import time
import re
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0, os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-from ruffus import (pipeline_run, pipeline_printout, suffix, transform, split,
- merge, dbdict, follows)
-from ruffus.combinatorics import *
-from ruffus.ruffus_exceptions import RethrownJobError
-from ruffus.ruffus_utility import (RUFFUS_HISTORY_FILE,
- CHECKSUM_FILE_TIMESTAMPS)
-workdir = 'tmp_test_combinatorics'
-#sub-1s resolution in system?
-one_second_per_job = None
-
-def touch (filename):
- with open(filename, "w"):
+def touch (outfile):
+ with open(outfile, "w"):
pass
@@ -44,45 +62,27 @@ def touch (filename):
#
# generate_initial_files1
#___________________________________________________________________________
- at originate([workdir + "/" + prefix + "_name.tmp1" for prefix in "abcd"])
-def generate_initial_files1(out_name):
- with open(out_name, 'w') as outfile:
- pass
+def generate_initial_files1(outfile):
+ touch(outfile)
#___________________________________________________________________________
#
-# generate_initial_files1
+# generate_initial_files2
#___________________________________________________________________________
- at originate([workdir + "/e_name.tmp1", workdir + "/f_name.tmp1"])
-def generate_initial_files2(out_name):
- with open(out_name, 'w') as outfile:
- pass
+def generate_initial_files2(outfile):
+ touch(outfile)
#___________________________________________________________________________
#
-# generate_initial_files1
+# generate_initial_files3
#___________________________________________________________________________
- at originate([workdir + "/g_name.tmp1", workdir + "/h_name.tmp1"])
-def generate_initial_files3(out_name):
- with open(out_name, 'w') as outfile:
- pass
+def generate_initial_files3(outfile):
+ touch(outfile)
#___________________________________________________________________________
#
# test_product_task
#___________________________________________________________________________
- at follows(generate_initial_files1)
- at product(
- [workdir + "/" + prefix + "_name.tmp1" for prefix in "abcd"],
- formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
- generate_initial_files2,
- formatter(),
- generate_initial_files3,
- formatter(r"tmp1$" ),
- "{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.{basename[2][0]}.tmp2",
- "{basename[0][0][0]}{basename[1][0][0]}{basename[2][0][0]}", # extra: prefices only (abcd etc)
- "{subpath[0][0][0]}", # extra: path for 2nd input, 1st file
- "{subdir[0][0][0]}")
def test_product_task( infiles, outfile,
prefices,
subpath,
@@ -95,7 +95,6 @@ def test_product_task( infiles, outfile,
#
# test_product_merged_task
#___________________________________________________________________________
- at merge(test_product_task, workdir + "/merged.results")
def test_product_merged_task( infiles, outfile):
with open(outfile, "w") as p:
for infile in sorted(infiles):
@@ -106,59 +105,37 @@ def test_product_merged_task( infiles, outfile):
#
# test_product_misspelt_capture_error_task
#___________________________________________________________________________
- at product(
- generate_initial_files1,
- formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
- "{path[0][0]}/{FILEPART[0][0]}.tmp2")
def test_product_misspelt_capture_error_task( infiles, outfile):
"""
FILE_PART mispelt as FILE_PART
"""
- with open(outfile, "w") as p: pass
+ touch(outfile)
#___________________________________________________________________________
#
# test_product_out_of_range_formatter_ref_error_task
#___________________________________________________________________________
- at product(
- generate_initial_files1,
- formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
- "{path[2][0]}/{basename[0][0]}.tmp2",
- "{FILE_PART[0][0]}")
def test_product_out_of_range_formatter_ref_error_task( infiles, outfile, ignored_filter):
"""
{path[2][0]} when len(path) == 1
"""
- with open(outfile, "w") as p: pass
+ touch(outfile)
#___________________________________________________________________________
#
# test_product_formatter_ref_index_error_task
#___________________________________________________________________________
- at product(
- generate_initial_files1,
- formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
- "{path[0][0][1000]}/{basename[0][0]}.tmp2",
- "{FILE_PART[0][0]}")
def test_product_formatter_ref_index_error_task( infiles, outfile, ignored_filter):
"""
{path[0][0][1000} when len of the path string len(path[0][0]) < 1000
"""
- with open(outfile, "w") as p: pass
+ touch(outfile)
#___________________________________________________________________________
#
# test_combinations2_task
#___________________________________________________________________________
- at combinations(
- generate_initial_files1,
- formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
- 2,
- "{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.tmp2",
- "{basename[0][0][0]}{basename[1][0][0]}", # extra: prefices
- "{subpath[0][0][0]}", # extra: path for 2nd input, 1st file
- "{subdir[0][0][0]}")
def test_combinations2_task( infiles, outfile,
prefices,
subpath,
@@ -170,7 +147,6 @@ def test_combinations2_task( infiles, outfile,
outf.write(prefices + ",")
- at merge(test_combinations2_task, workdir + "/merged.results")
def test_combinations2_merged_task( infiles, outfile):
with open(outfile, "w") as p:
for infile in sorted(infiles):
@@ -181,14 +157,6 @@ def test_combinations2_merged_task( infiles, outfile):
#
# test_combinations3_task
#___________________________________________________________________________
- at combinations(
- generate_initial_files1,
- formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
- 3,
- "{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.{basename[2][0]}.tmp2",
- "{basename[0][0][0]}{basename[1][0][0]}{basename[2][0][0]}", # extra: prefices
- "{subpath[0][0][0]}", # extra: path for 2nd input, 1st file
- "{subdir[0][0][0]}")
def test_combinations3_task( infiles, outfile,
prefices,
subpath,
@@ -199,7 +167,6 @@ def test_combinations3_task( infiles, outfile,
with open(outfile, "w") as outf:
outf.write(prefices + ",")
- at merge(test_combinations3_task, workdir + "/merged.results")
def test_combinations3_merged_task( infiles, outfile):
with open(outfile, "w") as p:
for infile in sorted(infiles):
@@ -211,14 +178,6 @@ def test_combinations3_merged_task( infiles, outfile):
#
# test_permutations2_task
#___________________________________________________________________________
- at permutations(
- generate_initial_files1,
- formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
- 2,
- "{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.tmp2",
- "{basename[0][0][0]}{basename[1][0][0]}", # extra: prefices
- "{subpath[0][0][0]}", # extra: path for 2nd input, 1st file
- "{subdir[0][0][0]}")
def test_permutations2_task( infiles, outfile,
prefices,
subpath,
@@ -229,7 +188,6 @@ def test_permutations2_task( infiles, outfile,
with open(outfile, "w") as outf:
outf.write(prefices + ",")
- at merge(test_permutations2_task, workdir + "/merged.results")
def test_permutations2_merged_task( infiles, outfile):
with open(outfile, "w") as p:
for infile in sorted(infiles):
@@ -241,14 +199,6 @@ def test_permutations2_merged_task( infiles, outfile):
#
# test_permutations3_task
#___________________________________________________________________________
- at permutations(
- generate_initial_files1,
- formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
- 3,
- "{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.{basename[2][0]}.tmp2",
- "{basename[0][0][0]}{basename[1][0][0]}{basename[2][0][0]}", # extra: prefices
- "{subpath[0][0][0]}", # extra: path for 2nd input, 1st file
- "{subdir[0][0][0]}")
def test_permutations3_task( infiles, outfile,
prefices,
subpath,
@@ -259,7 +209,6 @@ def test_permutations3_task( infiles, outfile,
with open(outfile, "w") as outf:
outf.write(prefices + ",")
- at merge(test_permutations3_task, workdir + "/merged.results")
def test_permutations3_merged_task( infiles, outfile):
with open(outfile, "w") as p:
for infile in sorted(infiles):
@@ -272,14 +221,6 @@ def test_permutations3_merged_task( infiles, outfile):
#
# test_combinations_with_replacement2_task
#___________________________________________________________________________
- at combinations_with_replacement(
- generate_initial_files1,
- formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
- 2,
- "{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.tmp2",
- "{basename[0][0][0]}{basename[1][0][0]}", # extra: prefices
- "{subpath[0][0][0]}", # extra: path for 2nd input, 1st file
- "{subdir[0][0][0]}")
def test_combinations_with_replacement2_task( infiles, outfile,
prefices,
subpath,
@@ -290,7 +231,6 @@ def test_combinations_with_replacement2_task( infiles, outfile,
with open(outfile, "w") as outf:
outf.write(prefices + ",")
- at merge(test_combinations_with_replacement2_task, workdir + "/merged.results")
def test_combinations_with_replacement2_merged_task( infiles, outfile):
with open(outfile, "w") as p:
for infile in sorted(infiles):
@@ -302,14 +242,6 @@ def test_combinations_with_replacement2_merged_task( infiles, outfile):
#
# test_combinations_with_replacement3_task
#___________________________________________________________________________
- at combinations_with_replacement(
- generate_initial_files1,
- formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
- 3,
- "{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.{basename[2][0]}.tmp2",
- "{basename[0][0][0]}{basename[1][0][0]}{basename[2][0][0]}", # extra: prefices
- "{subpath[0][0][0]}", # extra: path for 2nd input, 1st file
- "{subdir[0][0][0]}")
def test_combinations_with_replacement3_task( infiles, outfile,
prefices,
subpath,
@@ -320,7 +252,6 @@ def test_combinations_with_replacement3_task( infiles, outfile,
with open(outfile, "w") as outf:
outf.write(prefices + ",")
- at merge(test_combinations_with_replacement3_task, workdir + "/merged.results")
def test_combinations_with_replacement3_merged_task( infiles, outfile):
with open(outfile, "w") as p:
for infile in sorted(infiles):
@@ -333,6 +264,108 @@ def cleanup_tmpdir():
os.system('rm -f %s %s' % (os.path.join(workdir, '*'), RUFFUS_HISTORY_FILE))
+test_pipeline1 = Pipeline("test1")
+test_pipeline2 = Pipeline("test2")
+gen_task1 = test_pipeline1.originate(task_func = generate_initial_files1,
+ name = "WOWWWEEE",
+ output = [workdir + "/" + prefix + "_name.tmp1" for prefix in "abcd"])
+test_pipeline1.originate( task_func = generate_initial_files2,
+ output = [workdir + "/e_name.tmp1", workdir + "/f_name.tmp1"])
+test_pipeline1.originate( task_func = generate_initial_files3,
+ output = [workdir + "/g_name.tmp1", workdir + "/h_name.tmp1"])
+test_pipeline1.product( task_func = test_product_task,
+ input = [workdir + "/" + prefix + "_name.tmp1" for prefix in "abcd"],
+ filter = formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
+ input2 = generate_initial_files2,
+ filter2 = formatter(),
+ input3 = generate_initial_files3,
+ filter3 = formatter(r"tmp1$" ),
+ output = "{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.{basename[2][0]}.tmp2",
+ extras = [ "{basename[0][0][0]}{basename[1][0][0]}{basename[2][0][0]}", # extra: prefices only (abcd etc)
+ "{subpath[0][0][0]}", # extra: path for 2nd input, 1st file
+ "{subdir[0][0][0]}"]).follows("WOWWWEEE").follows(gen_task1).follows(generate_initial_files1).follows("generate_initial_files1")
+test_pipeline1.merge( task_func = test_product_merged_task,
+ input = test_product_task,
+ output = workdir + "/merged.results")
+test_pipeline1.product( task_func = test_product_misspelt_capture_error_task,
+ input = gen_task1,
+ filter = formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
+ output = "{path[0][0]}/{FILEPART[0][0]}.tmp2")
+test_pipeline1.product( task_func = test_product_out_of_range_formatter_ref_error_task,
+ input = generate_initial_files1, #
+ filter = formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
+ output = "{path[2][0]}/{basename[0][0]}.tmp2",
+ extras = ["{FILE_PART[0][0]}"])
+test_pipeline1.product( task_func = test_product_formatter_ref_index_error_task,
+ input = output_from("generate_initial_files1"),
+ filter = formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
+ output = "{path[0][0][1000]}/{basename[0][0]}.tmp2",
+ extras = ["{FILE_PART[0][0]}"])
+test_pipeline1.combinations(task_func = test_combinations2_task,
+ input = generate_initial_files1, # gen_task1
+ filter = formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
+ tuple_size = 2,
+ output = "{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.tmp2",
+ extras = ["{basename[0][0][0]}{basename[1][0][0]}", # extra: prefices
+ "{subpath[0][0][0]}", # extra: path for 2nd input, 1st file
+ "{subdir[0][0][0]}"])
+test_pipeline1.merge(task_func = test_combinations2_merged_task,
+ input = test_combinations2_task,
+ output = workdir + "/merged.results")
+test_pipeline1.combinations( task_func = test_combinations3_task,
+ input = output_from("WOWWWEEE"),
+ filter = formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
+ tuple_size = 3,
+ output = "{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.{basename[2][0]}.tmp2",
+ extras = ["{basename[0][0][0]}{basename[1][0][0]}{basename[2][0][0]}", # extra: prefices
+ "{subpath[0][0][0]}", # extra: path for 2nd input, 1st file
+ "{subdir[0][0][0]}"])
+
+test_pipeline1.merge(test_combinations3_merged_task, test_combinations3_task, workdir + "/merged.results")
+test_pipeline1.permutations(task_func = test_permutations2_task,
+ input = output_from("WOWWWEEE"),
+ filter = formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
+ tuple_size = 2,
+ output = "{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.tmp2",
+ extras = ["{basename[0][0][0]}{basename[1][0][0]}", # extra: prefices
+ "{subpath[0][0][0]}", # extra: path for 2nd input, 1st file
+ "{subdir[0][0][0]}"])
+
+test_pipeline2.merge(test_permutations2_merged_task, test_permutations2_task, workdir + "/merged.results")
+
+
+
+test_pipeline2.permutations(task_func = test_permutations3_task,
+ input = output_from("WOWWWEEE"),
+ filter = formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
+ tuple_size = 3,
+ output = "{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.{basename[2][0]}.tmp2",
+ extras = ["{basename[0][0][0]}{basename[1][0][0]}{basename[2][0][0]}", # extra: prefices
+ "{subpath[0][0][0]}", # extra: path for 2nd input, 1st file
+ "{subdir[0][0][0]}"])
+test_pipeline2.merge(test_permutations3_merged_task, test_permutations3_task, workdir + "/merged.results")
+test_pipeline2.combinations_with_replacement(test_combinations_with_replacement2_task,
+ input = output_from("WOWWWEEE"),
+ filter = formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
+ tuple_size = 2,
+ output = "{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.tmp2",
+ extras = ["{basename[0][0][0]}{basename[1][0][0]}", # extra: prefices
+ "{subpath[0][0][0]}", # extra: path for 2nd input, 1st file
+ "{subdir[0][0][0]}"])
+test_pipeline2.merge(test_combinations_with_replacement2_merged_task,
+ test_combinations_with_replacement2_task, workdir + "/merged.results")
+test_pipeline2.combinations_with_replacement( task_func = test_combinations_with_replacement3_task,
+ input = output_from("WOWWWEEE"),
+ filter = formatter(".*/(?P<FILE_PART>.+).tmp1$" ),
+ tuple_size = 3,
+ output = "{path[0][0]}/{FILE_PART[0][0]}.{basename[1][0]}.{basename[2][0]}.tmp2",
+ extras = ["{basename[0][0][0]}{basename[1][0][0]}{basename[2][0][0]}", # extra: prefices
+ "{subpath[0][0][0]}", # extra: path for 2nd input, 1st file
+ "{subdir[0][0][0]}"])
+test_pipeline2.merge(test_combinations_with_replacement3_merged_task,
+ test_combinations_with_replacement3_task, workdir + "/merged.results")
+
+
class TestCombinatorics(unittest.TestCase):
def setUp(self):
try:
@@ -348,18 +381,18 @@ class TestCombinatorics(unittest.TestCase):
"""Input file exists, output doesn't exist"""
cleanup_tmpdir()
s = StringIO()
- pipeline_printout(s, [test_product_merged_task], verbose=5, wrap_width = 10000)
- self.assertTrue(re.search('Job needs update: Missing files\n\s+'
+ test_pipeline2.printout(s, [test_product_merged_task], verbose=5, wrap_width = 10000)
+ self.assertTrue(re.search('Job needs update:.*Missing files.*'
'\[.*tmp_test_combinatorics/a_name.tmp1, '
'.*tmp_test_combinatorics/e_name.tmp1, '
'.*tmp_test_combinatorics/h_name.tmp1, '
- '.*tmp_test_combinatorics/a_name.e_name.h_name.tmp2\]', s.getvalue()))
+ '.*tmp_test_combinatorics/a_name.e_name.h_name.tmp2\]', s.getvalue(), re.DOTALL))
def test_product_run(self):
"""Run product"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
- pipeline_run([test_product_merged_task], verbose=0, multiprocess = 100, one_second_per_job = one_second_per_job)
+ test_pipeline2.run([test_product_merged_task], verbose=0, multiprocess = 100, one_second_per_job = one_second_per_job)
with open(workdir + "/merged.results") as oo:
self.assertEqual(oo.read(),
"aeg,aeh,afg,afh,beg,beh,bfg,bfh,ceg,ceh,cfg,cfh,deg,deh,dfg,dfh,")
@@ -377,7 +410,7 @@ class TestCombinatorics(unittest.TestCase):
cleanup_tmpdir()
s = StringIO()
- pipeline_printout(s, [test_product_misspelt_capture_error_task], verbose=3, wrap_width = 10000)
+ test_pipeline2.printout(s, [test_product_misspelt_capture_error_task], verbose=3, wrap_width = 10000)
self.assertIn("Warning: File match failure: Unmatched field 'FILEPART'", s.getvalue())
@@ -389,7 +422,7 @@ class TestCombinatorics(unittest.TestCase):
cleanup_tmpdir()
s = StringIO()
- pipeline_printout(s, [test_product_out_of_range_formatter_ref_error_task], verbose=3, wrap_width = 10000)
+ test_pipeline2.printout(s, [test_product_out_of_range_formatter_ref_error_task], verbose=3, wrap_width = 10000)
self.assertIn("Warning: File match failure: Unmatched field 2", s.getvalue())
def test_product_formatter_ref_index_error(self):
@@ -400,7 +433,7 @@ class TestCombinatorics(unittest.TestCase):
cleanup_tmpdir()
s = StringIO()
- pipeline_printout(s, [test_product_formatter_ref_index_error_task], verbose=3, wrap_width = 10000)
+ test_pipeline2.printout(s, [test_product_formatter_ref_index_error_task], verbose=3, wrap_width = 10000)
self.assertIn("Warning: File match failure: Unmatched field string index out of range", s.getvalue())
#print s.getvalue()
@@ -414,18 +447,18 @@ class TestCombinatorics(unittest.TestCase):
cleanup_tmpdir()
s = StringIO()
- pipeline_printout(s, [test_combinations2_merged_task], verbose=5, wrap_width = 10000)
- self.assertTrue(re.search('Job needs update: Missing files\n\s+'
+ test_pipeline1.printout(s, [test_combinations2_merged_task], verbose=5, wrap_width = 10000)
+ self.assertTrue(re.search('Job needs update:.*Missing files.*'
'\[.*tmp_test_combinatorics/a_name.tmp1, '
'.*tmp_test_combinatorics/b_name.tmp1, '
- '.*tmp_test_combinatorics/a_name.b_name.tmp2\]', s.getvalue()))
+ '.*tmp_test_combinatorics/a_name.b_name.tmp2\]', s.getvalue(), re.DOTALL))
def test_combinations2_run(self):
"""Run product"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
- pipeline_run([test_combinations2_merged_task], verbose=0, multiprocess = 100, one_second_per_job = one_second_per_job)
+ test_pipeline2.run([test_combinations2_merged_task], verbose=0, multiprocess = 100, one_second_per_job = one_second_per_job)
with open(workdir + "/merged.results") as oo:
self.assertEqual(oo.read(),
'ab,ac,ad,bc,bd,cd,')
@@ -439,7 +472,7 @@ class TestCombinatorics(unittest.TestCase):
cleanup_tmpdir()
s = StringIO()
- pipeline_printout(s, [test_combinations3_merged_task], verbose=5, wrap_width = 10000)
+ test_pipeline2.printout(s, [test_combinations3_merged_task], verbose=5, wrap_width = 10000)
self.assertTrue(re.search(
'\[.*tmp_test_combinatorics/a_name.tmp1, '
'.*tmp_test_combinatorics/b_name.tmp1, '
@@ -450,7 +483,7 @@ class TestCombinatorics(unittest.TestCase):
"""Run product"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
- pipeline_run([test_combinations3_merged_task], verbose=0, multiprocess = 100, one_second_per_job = one_second_per_job)
+ test_pipeline2.run([test_combinations3_merged_task], verbose=0, multiprocess = 100, one_second_per_job = one_second_per_job)
with open(workdir + "/merged.results") as oo:
self.assertEqual(oo.read(),
"abc,abd,acd,bcd,")
@@ -465,7 +498,7 @@ class TestCombinatorics(unittest.TestCase):
cleanup_tmpdir()
s = StringIO()
- pipeline_printout(s, [test_permutations2_merged_task], verbose=5, wrap_width = 10000)
+ test_pipeline2.printout(s, [test_permutations2_merged_task], verbose=5, wrap_width = 10000)
self.assertTrue(re.search('\[.*tmp_test_combinatorics/a_name.tmp1, '
'.*tmp_test_combinatorics/b_name.tmp1, '
'.*tmp_test_combinatorics/a_name.b_name.tmp2\]', s.getvalue()))
@@ -474,7 +507,7 @@ class TestCombinatorics(unittest.TestCase):
"""Run product"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
- pipeline_run([test_permutations2_merged_task], verbose=0, multiprocess = 100, one_second_per_job = one_second_per_job)
+ test_pipeline2.run([test_permutations2_merged_task], verbose=0, multiprocess = 100, one_second_per_job = one_second_per_job)
with open(workdir + "/merged.results") as oo:
self.assertEqual(oo.read(),
"ab,ac,ad,ba,bc,bd,ca,cb,cd,da,db,dc,")
@@ -488,7 +521,7 @@ class TestCombinatorics(unittest.TestCase):
cleanup_tmpdir()
s = StringIO()
- pipeline_printout(s, [test_permutations3_merged_task], verbose=5, wrap_width = 10000)
+ test_pipeline2.printout(s, [test_permutations3_merged_task], verbose=5, wrap_width = 10000)
self.assertTrue(re.search('\[.*tmp_test_combinatorics/a_name.tmp1, '
'.*tmp_test_combinatorics/b_name.tmp1, '
'.*tmp_test_combinatorics/c_name.tmp1, '
@@ -498,7 +531,7 @@ class TestCombinatorics(unittest.TestCase):
"""Run product"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
- pipeline_run([test_permutations3_merged_task], verbose=0, multiprocess = 100, one_second_per_job = one_second_per_job)
+ test_pipeline2.run([test_permutations3_merged_task], verbose=0, multiprocess = 100, one_second_per_job = one_second_per_job)
with open(workdir + "/merged.results") as oo:
self.assertEqual(oo.read(),
'abc,abd,acb,acd,adb,adc,bac,bad,bca,bcd,bda,bdc,cab,cad,cba,cbd,cda,cdb,dab,dac,dba,dbc,dca,dcb,')
@@ -513,7 +546,7 @@ class TestCombinatorics(unittest.TestCase):
cleanup_tmpdir()
s = StringIO()
- pipeline_printout(s, [test_combinations_with_replacement2_merged_task], verbose=5, wrap_width = 10000)
+ test_pipeline2.printout(s, [test_combinations_with_replacement2_merged_task], verbose=5, wrap_width = 10000)
self.assertTrue(re.search('\[.*tmp_test_combinatorics/a_name.tmp1, '
'.*tmp_test_combinatorics/b_name.tmp1, '
'.*tmp_test_combinatorics/a_name.b_name.tmp2\]', s.getvalue()))
@@ -522,7 +555,7 @@ class TestCombinatorics(unittest.TestCase):
"""Run product"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
- pipeline_run([test_combinations_with_replacement2_merged_task], verbose=0, multiprocess = 100, one_second_per_job = one_second_per_job)
+ test_pipeline2.run([test_combinations_with_replacement2_merged_task], verbose=0, multiprocess = 100, one_second_per_job = one_second_per_job)
with open(workdir + "/merged.results") as oo:
self.assertEqual(oo.read(),
"aa,ab,ac,ad,bb,bc,bd,cc,cd,dd,")
@@ -536,7 +569,7 @@ class TestCombinatorics(unittest.TestCase):
cleanup_tmpdir()
s = StringIO()
- pipeline_printout(s, [test_combinations_with_replacement3_merged_task], verbose=5, wrap_width = 10000)
+ test_pipeline2.printout(s, [test_combinations_with_replacement3_merged_task], verbose=5, wrap_width = 10000)
self.assertTrue(re.search('\[.*tmp_test_combinatorics/a_name.tmp1, '
'.*tmp_test_combinatorics/b_name.tmp1, '
'.*tmp_test_combinatorics/c_name.tmp1, '
@@ -546,7 +579,7 @@ class TestCombinatorics(unittest.TestCase):
"""Run product"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
- pipeline_run([test_combinations_with_replacement3_merged_task], verbose=0, multiprocess = 100, one_second_per_job = one_second_per_job)
+ test_pipeline2.run([test_combinations_with_replacement3_merged_task], verbose=0, multiprocess = 100, one_second_per_job = one_second_per_job)
with open(workdir + "/merged.results") as oo:
self.assertEqual(oo.read(),
'aaa,aab,aac,aad,abb,abc,abd,acc,acd,add,bbb,bbc,bbd,bcc,bcd,bdd,ccc,ccd,cdd,ddd,')
@@ -557,7 +590,6 @@ class TestCombinatorics(unittest.TestCase):
# cleanup
#___________________________________________________________________________
def tearDown(self):
- pass
shutil.rmtree(workdir)
diff --git a/ruffus/test/test_newstyle_proxy.py b/ruffus/test/test_newstyle_proxy.py
new file mode 100755
index 0000000..e47a383
--- /dev/null
+++ b/ruffus/test/test_newstyle_proxy.py
@@ -0,0 +1,181 @@
+#!/usr/bin/env python
+from __future__ import print_function
+"""
+
+ test_softlink_uptodate.py
+
+"""
+
+import os
+import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+try:
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
+
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# Tasks
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+
+import multiprocessing.managers
+
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# Tasks
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+#
+# First task
+#
+def start_task(output_file_name, executed_tasks_proxy, mutex_proxy):
+ with open(output_file_name, "w") as f:
+ pass
+ with mutex_proxy:
+ executed_tasks_proxy["start_task"] = 1
+
+#
+# Forwards file names, is always as up to date as its input files...
+#
+def same_file_name_task(input_file_name, output_file_name, executed_tasks_proxy, mutex_proxy):
+ with mutex_proxy:
+ executed_tasks_proxy["same_file_name_task"] = executed_tasks_proxy.get("same_file_name_task", 0) + 1
+
+#
+# Links file names, is always as up to date if links are not missing
+#
+def linked_file_name_task(input_file_name, output_file_name, executed_tasks_proxy, mutex_proxy):
+ os.symlink(input_file_name, output_file_name)
+ with mutex_proxy:
+ executed_tasks_proxy["linked_file_name_task"] = executed_tasks_proxy.get("linked_file_name_task", 0) + 1
+
+
+#
+# Final task linking everything
+#
+def final_task (input_file_name, output_file_name, executed_tasks_proxy, mutex_proxy):
+ with open(output_file_name, "w") as f:
+ pass
+ with mutex_proxy:
+ executed_tasks_proxy["final_task"] = executed_tasks_proxy.get("final_task", 0) + 1
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# Run pipeline
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+
+import unittest, shutil
+try:
+ from StringIO import StringIO
+except:
+ from io import StringIO
+
+class Test_ruffus(unittest.TestCase):
+ def setUp(self):
+
+ # list of executed tasks
+ manager = multiprocessing.managers.SyncManager()
+ manager.start()
+ global mutex_proxy
+ global executed_tasks_proxy
+ mutex_proxy = manager.Lock()
+ executed_tasks_proxy = manager.dict()
+
+ pipeline = Pipeline.pipelines["main"]
+ pipeline.originate(task_func = start_task,
+ output = ["a.1", "b.1"],
+ extras = [executed_tasks_proxy, mutex_proxy])
+ pipeline.transform(task_func = same_file_name_task,
+ input = start_task,
+ filter = suffix(".1"),
+ output = ".1",
+ extras = [executed_tasks_proxy, mutex_proxy])
+ pipeline.transform( task_func = linked_file_name_task,
+ input = start_task,
+ filter = suffix(".1"),
+ output = ".linked.1",
+ extras = [executed_tasks_proxy, mutex_proxy])
+ pipeline.transform(task_func = final_task,
+ input = [linked_file_name_task, same_file_name_task],
+ filter = suffix(".1"),
+ output = ".3",
+ extras = [executed_tasks_proxy, mutex_proxy])
+ self.cleanUp()
+
+ def cleanUp(self, check_expected = False):
+ for f in ["a.1", "b.1", "a.linked.1", "b.linked.1", "a.3", "b.3", "a.linked.3", "b.linked.3"]:
+ if os.path.lexists(f):
+ os.unlink(f)
+ elif check_expected:
+ raise Exception("Expected %s missing" % f)
+
+ def tearDown(self):
+ self.cleanUp(True)
+
+ def test_ruffus (self):
+ #
+ # Run task 1 only
+ #
+ print(" Run start_task only", file=sys.stderr)
+ pipeline_run(log_exceptions = True, verbose = 0)
+
+
+ #
+ # Run task 3 only
+ #
+ print(" Run final_task: linked_file_name_task should run as well", file=sys.stderr)
+ pipeline_run(log_exceptions = True, verbose = 0)
+
+
+ #
+ # Run task 3 again:
+ #
+ # All jobs should be up to date
+ #
+ print(" Run final_task again: All jobs should be up to date", file=sys.stderr)
+ pipeline_run(log_exceptions = True, verbose = 0)
+
+ #
+ # Make sure right number of jobs / tasks ran
+ #
+ for task_name, jobs_count in ({'start_task': 1, 'final_task': 4, 'linked_file_name_task': 2}).items():
+ if task_name not in executed_tasks_proxy:
+ raise Exception("Error: %s did not run!!" % task_name)
+ if executed_tasks_proxy[task_name] != jobs_count:
+ raise Exception("Error: %s did not have %d jobs!!" % (task_name, jobs_count))
+ if "same_file_name_task" in executed_tasks_proxy:
+ raise Exception("Error: %s should not have run!!" % "same_file_name_task")
+
+
+
+if __name__ == '__main__':
+ unittest.main()
+
diff --git a/ruffus/test/test_regex_error_messages.py b/ruffus/test/test_newstyle_regex_error_messages.py
similarity index 68%
copy from ruffus/test/test_regex_error_messages.py
copy to ruffus/test/test_newstyle_regex_error_messages.py
index 796aa7c..ce72ab0 100755
--- a/ruffus/test/test_regex_error_messages.py
+++ b/ruffus/test/test_newstyle_regex_error_messages.py
@@ -28,34 +28,48 @@ from __future__ import print_function
"""
+workdir = 'tmp_test_regex_error_messages'
+#sub-1s resolution in system?
+one_second_per_job = None
+parallelism = 2
-import unittest
-import os, re
+
+import os
import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+for attr in "pipeline_run", "pipeline_printout", "suffix", "transform", "split", "merge", "dbdict", "follows", "originate", "Pipeline", "regex":
+ globals()[attr] = getattr (ruffus, attr)
+RUFFUS_HISTORY_FILE = ruffus.ruffus_utility.RUFFUS_HISTORY_FILE
+fatal_error_input_file_does_not_match = ruffus.ruffus_exceptions.fatal_error_input_file_does_not_match
+RethrownJobError = ruffus.ruffus_exceptions.RethrownJobError
+
+
+
+import unittest
import shutil
+import re
try:
from StringIO import StringIO
except:
from io import StringIO
-import time
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0, os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-from ruffus import (pipeline_run, pipeline_printout, suffix, transform, split,
- merge, dbdict, follows)
-from ruffus.ruffus_exceptions import *
-from ruffus.ruffus_utility import (RUFFUS_HISTORY_FILE)
-workdir = 'tmp_test_regex_error_messages'
-#sub-1s resolution in system?
-one_second_per_job = None
-parallelism = 2
#___________________________________________________________________________
#
# generate_initial_files1
#___________________________________________________________________________
- at originate([workdir + "/" + prefix + "_name.tmp1" for prefix in "abcdefghi"])
def generate_initial_files1(out_name):
with open(out_name, 'w') as outfile:
pass
@@ -64,13 +78,6 @@ def generate_initial_files1(out_name):
#
# test_regex_task
#___________________________________________________________________________
- at transform(
- generate_initial_files1,
- regex("(.*)/(?P<PREFIX>[abcd])(_name)(.tmp1)"),
- r"\1/\g<PREFIX>\3.tmp2",# output file
- r"\2", # extra: prefix = \2
- r"\g<PREFIX>", # extra: prefix = \2
- r"\4") # extra: extension
def test_regex_task(infiles, outfile,
prefix1,
prefix2,
@@ -86,13 +93,6 @@ def test_regex_task(infiles, outfile,
#
# test_regex_unmatched_task
#___________________________________________________________________________
- at transform(
- generate_initial_files1,
- regex("(.*)/(?P<PREFIX>[abcd])(_name)(.xxx)"),
- r"\1/\g<PREFIXA>\3.tmp2",# output file
- r"\2", # extra: prefix = \2
- r"\g<PREFIX>", # extra: prefix = \2
- r"\4") # extra: extension
def test_regex_unmatched_task(infiles, outfile,
prefix1,
prefix2,
@@ -104,11 +104,6 @@ def test_regex_unmatched_task(infiles, outfile,
#
# test_suffix_task
#___________________________________________________________________________
- at transform(
- generate_initial_files1,
- suffix(".tmp1"),
- r".tmp2", # output file
- r"\1") # extra: basename
def test_suffix_task(infile, outfile,
basename):
with open (outfile, "w") as f: pass
@@ -118,11 +113,6 @@ def test_suffix_task(infile, outfile,
#
# test_suffix_unmatched_task
#___________________________________________________________________________
- at transform(
- generate_initial_files1,
- suffix(".tmp1"),
- r".tmp2", # output file
- r"\2") # extra: unknown
def test_suffix_unmatched_task(infiles, outfile, unknown):
raise Exception("Should blow up first")
@@ -131,10 +121,6 @@ def test_suffix_unmatched_task(infiles, outfile, unknown):
#
# test_suffix_unmatched_task
#___________________________________________________________________________
- at transform(
- generate_initial_files1,
- suffix(".tmp2"),
- r".tmp2") # output file
def test_suffix_unmatched_task2(infiles, outfile):
raise Exception("Should blow up first")
@@ -144,13 +130,6 @@ def test_suffix_unmatched_task2(infiles, outfile):
#
# test_product_misspelt_capture_error_task
#___________________________________________________________________________
- at transform(
- generate_initial_files1,
- regex("(.*)/(?P<PREFIX>[abcd])(_name)(.tmp)"),
- r"\1/\g<PREFIXA>\3.tmp2",# output file
- r"\2", # extra: prefix = \2
- r"\g<PREFIX>", # extra: prefix = \2
- r"\4") # extra: extension
def test_regex_misspelt_capture_error_task(infiles, outfile,
prefix1,
prefix2,
@@ -162,13 +141,6 @@ def test_regex_misspelt_capture_error_task(infiles, outfile,
#
# test_regex_misspelt_capture2_error_task
#___________________________________________________________________________
- at transform(
- generate_initial_files1,
- regex("(.*)/(?P<PREFIX>[abcd])(_name)(.tmp)"),
- r"\1/\g<PREFIX>\3.tmp2",# output file
- r"\2", # extra: prefix = \2
- r"\g<PREFIXA>", # extra: prefix = \2
- r"\4") # extra: extension
def test_regex_misspelt_capture2_error_task(infiles, outfile,
prefix1,
prefix2,
@@ -180,13 +152,6 @@ def test_regex_misspelt_capture2_error_task(infiles, outfile,
#
# test_regex_out_of_range_regex_reference_error_task
#___________________________________________________________________________
- at transform(
- generate_initial_files1,
- regex("(.*)/(?P<PREFIX>[abcd])(_name)(.tmp)"),
- r"\1/\g<PREFIX>\5.tmp2",# output file
- r"\2", # extra: prefix = \2
- r"\g<PREFIX>", # extra: prefix = \2
- r"\4") # extra: extension
def test_regex_out_of_range_regex_reference_error_task(infiles, outfile,
prefix1,
prefix2,
@@ -197,6 +162,72 @@ def test_regex_out_of_range_regex_reference_error_task(infiles, outfile,
+
+
+
+test_pipeline = Pipeline("test")
+
+test_pipeline.originate(task_func = generate_initial_files1,
+ output = [workdir + "/" + prefix + "_name.tmp1" for prefix in "abcdefghi"])
+
+test_pipeline.transform(task_func = test_regex_task,
+ input = generate_initial_files1,
+ filter = regex("(.*)/(?P<PREFIX>[abcd])(_name)(.tmp1)"),
+ output = r"\1/\g<PREFIX>\3.tmp2",# output file
+ extras = [ r"\2", # extra: prefix = \2
+ r"\g<PREFIX>", # extra: prefix = \2
+ r"\4"]) # extra: extension
+test_pipeline.transform(task_func = test_regex_unmatched_task,
+ input = generate_initial_files1,
+ filter = regex("(.*)/(?P<PREFIX>[abcd])(_name)(.xxx)"),
+ output = r"\1/\g<PREFIXA>\3.tmp2",# output file
+ extras = [ r"\2", # extra: prefix = \2
+ r"\g<PREFIX>", # extra: prefix = \2
+ r"\4"]) # extra: extension
+
+test_pipeline.transform(task_func = test_suffix_task,
+ input = generate_initial_files1,
+ filter = suffix(".tmp1"),
+ output = r".tmp2", # output file
+ extras = [r"\1"]) # extra: basename
+
+test_pipeline.transform(task_func = test_suffix_unmatched_task,
+ input = generate_initial_files1,
+ filter = suffix(".tmp1"),
+ output = r".tmp2", # output file
+ extras = [r"\2"]) # extra: unknown
+
+test_pipeline.transform(task_func = test_suffix_unmatched_task2,
+ input = generate_initial_files1,
+ filter = suffix(".tmp2"),
+ output = r".tmp2") # output file
+
+test_pipeline.transform( task_func = test_regex_misspelt_capture_error_task,
+ input = generate_initial_files1,
+ filter = regex("(.*)/(?P<PREFIX>[abcd])(_name)(.tmp)"),
+ output = r"\1/\g<PREFIXA>\3.tmp2",# output file
+ extras = [r"\2", # extra: prefix = \2
+ r"\g<PREFIX>", # extra: prefix = \2
+ r"\4"]) # extra: extension
+test_pipeline.transform(task_func = test_regex_misspelt_capture2_error_task,
+ input = generate_initial_files1,
+ filter = regex("(.*)/(?P<PREFIX>[abcd])(_name)(.tmp)"),
+ output = r"\1/\g<PREFIX>\3.tmp2",# output file
+ extras = [r"\2", # extra: prefix = \2
+ r"\g<PREFIXA>", # extra: prefix = \2
+ r"\4"]) # extra: extension
+
+test_pipeline.transform(task_func = test_regex_out_of_range_regex_reference_error_task,
+ input = generate_initial_files1,
+ filter = regex("(.*)/(?P<PREFIX>[abcd])(_name)(.tmp)"),
+ output = r"\1/\g<PREFIX>\5.tmp2",# output file
+ extras = [r"\2", # extra: prefix = \2
+ r"\g<PREFIX>", # extra: prefix = \2
+ r"\4"]) # extra: extension
+
+
+
+
def cleanup_tmpdir():
os.system('rm -f %s %s' % (os.path.join(workdir, '*'), RUFFUS_HISTORY_FILE))
@@ -283,15 +314,15 @@ class Test_regex_error_messages(unittest.TestCase):
cleanup_tmpdir()
s = StringIO()
- pipeline_printout(s, [test_regex_task], verbose=5, wrap_width = 10000)
- self.assertTrue(re.search('Missing files\n\s+\[tmp_test_regex_error_messages/a_name.tmp1, tmp_test_regex_error_messages/a_name.tmp2', s.getvalue()))
+ test_pipeline.printout(s, [test_regex_task], verbose=5, wrap_width = 10000)
+ self.assertTrue(re.search('Missing files.*\[tmp_test_regex_error_messages/a_name.tmp1, tmp_test_regex_error_messages/a_name.tmp2', s.getvalue(), re.DOTALL))
def test_regex_run(self):
"""Run transform(...,regex()...)"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
- pipeline_run([test_regex_task], verbose=0, multiprocess = parallelism, one_second_per_job = one_second_per_job)
+ test_pipeline.run([test_regex_task], verbose=0, multiprocess = parallelism, one_second_per_job = one_second_per_job)
#___________________________________________________________________________
@@ -301,14 +332,14 @@ class Test_regex_error_messages(unittest.TestCase):
def test_regex_unmatched_printout(self):
cleanup_tmpdir()
s = StringIO()
- pipeline_printout(s, [test_regex_unmatched_task], verbose=5, wrap_width = 10000)
+ test_pipeline.printout(s, [test_regex_unmatched_task], verbose=5, wrap_width = 10000)
self.assertIn("Warning: File match failure: File 'tmp_test_regex_error_messages/a_name.tmp1' does not match regex", s.getvalue())
def test_regex_unmatched_run(self):
"""Run transform(...,regex()...)"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
- pipeline_run([test_regex_unmatched_task], verbose=0, multiprocess = parallelism, one_second_per_job = one_second_per_job)
+ test_pipeline.run([test_regex_unmatched_task], verbose=0, multiprocess = parallelism, one_second_per_job = one_second_per_job)
#___________________________________________________________________________
@@ -319,14 +350,14 @@ class Test_regex_error_messages(unittest.TestCase):
cleanup_tmpdir()
s = StringIO()
- pipeline_printout(s, [test_suffix_task], verbose=5, wrap_width = 10000)
- self.assertTrue(re.search('Missing files\n\s+\[tmp_test_regex_error_messages/a_name.tmp1, tmp_test_regex_error_messages/a_name.tmp2', s.getvalue()))
+ test_pipeline.printout(s, [test_suffix_task], verbose=5, wrap_width = 10000)
+ self.assertTrue(re.search('Missing files.*\[tmp_test_regex_error_messages/a_name.tmp1, tmp_test_regex_error_messages/a_name.tmp2', s.getvalue(), re.DOTALL))
def test_suffix_run(self):
"""Run transform(...,suffix()...)"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
- pipeline_run([test_suffix_task], verbose=0, multiprocess = parallelism, one_second_per_job = one_second_per_job)
+ test_pipeline.run([test_suffix_task], verbose=0, multiprocess = parallelism, one_second_per_job = one_second_per_job)
#___________________________________________________________________________
@@ -343,7 +374,7 @@ class Test_regex_error_messages(unittest.TestCase):
verbose = 3)
self.assertRaisesRegex(RethrownJobError,
"File '.*?' does not match regex\('.*?'\) and pattern '.*?':\n.*invalid group reference",
- pipeline_run,
+ test_pipeline.run,
[test_suffix_unmatched_task], verbose = 0, multiprocess = parallelism)
@@ -354,14 +385,14 @@ class Test_regex_error_messages(unittest.TestCase):
def test_suffix_unmatched_printout2(self):
cleanup_tmpdir()
s = StringIO()
- pipeline_printout(s, [test_suffix_unmatched_task2], verbose=5, wrap_width = 10000)
+ test_pipeline.printout(s, [test_suffix_unmatched_task2], verbose=5, wrap_width = 10000)
self.assertIn("Warning: File match failure: File 'tmp_test_regex_error_messages/a_name.tmp1' does not match suffix", s.getvalue())
def test_suffix_unmatched_run2(self):
"""Run transform(...,suffix()...)"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
- pipeline_run([test_suffix_unmatched_task2], verbose=0, multiprocess = parallelism, one_second_per_job = one_second_per_job)
+ test_pipeline.run([test_suffix_unmatched_task2], verbose=0, multiprocess = parallelism, one_second_per_job = one_second_per_job)
@@ -374,12 +405,12 @@ class Test_regex_error_messages(unittest.TestCase):
s = StringIO()
self.assertRaisesRegex(fatal_error_input_file_does_not_match,
"File '.*?' does not match regex\('.*?'\) and pattern '.*?':\n.*unknown group name",
- pipeline_printout,
+ test_pipeline.printout,
s, [test_regex_misspelt_capture_error_task],
verbose = 3)
self.assertRaisesRegex(RethrownJobError,
"File '.*?' does not match regex\('.*?'\) and pattern '.*?':\n.*unknown group name",
- pipeline_run,
+ test_pipeline.run,
[test_regex_misspelt_capture_error_task], verbose = 0)
#___________________________________________________________________________
@@ -391,12 +422,12 @@ class Test_regex_error_messages(unittest.TestCase):
s = StringIO()
self.assertRaisesRegex(fatal_error_input_file_does_not_match,
"File '.*?' does not match regex\('.*?'\) and pattern '.*?':\n.*unknown group name",
- pipeline_printout,
+ test_pipeline.printout,
s, [test_regex_misspelt_capture2_error_task],
verbose = 3)
self.assertRaisesRegex(RethrownJobError,
"File '.*?' does not match regex\('.*?'\) and pattern '.*?':\n.*unknown group name",
- pipeline_run,
+ test_pipeline.run,
[test_regex_misspelt_capture2_error_task], verbose = 0, multiprocess = parallelism)
@@ -409,12 +440,12 @@ class Test_regex_error_messages(unittest.TestCase):
s = StringIO()
self.assertRaisesRegex(fatal_error_input_file_does_not_match,
"File '.*?' does not match regex\('.*?'\) and pattern '.*?':\n.*invalid group reference",
- pipeline_printout,
+ test_pipeline.printout,
s, [test_regex_out_of_range_regex_reference_error_task],
verbose = 3)
self.assertRaisesRegex(RethrownJobError,
"File '.*?' does not match regex\('.*?'\) and pattern '.*?':\n.*invalid group reference",
- pipeline_run,
+ test_pipeline.run,
[test_regex_out_of_range_regex_reference_error_task], verbose = 0, multiprocess = parallelism)
@@ -436,9 +467,9 @@ if __name__ == '__main__':
#pipeline_printout(sys.stdout, [test_product_task], verbose = 3)
parallelism = 1
suite = unittest.TestLoader().loadTestsFromTestCase(Test_regex_error_messages)
- unittest.TextTestRunner(verbosity=2).run(suite)
+ unittest.TextTestRunner(verbosity=1).run(suite)
parallelism = 2
suite = unittest.TestLoader().loadTestsFromTestCase(Test_regex_error_messages)
- unittest.TextTestRunner(verbosity=2).run(suite)
+ unittest.TextTestRunner(verbosity=1).run(suite)
#unittest.main()
diff --git a/ruffus/test/test_pausing.py b/ruffus/test/test_pausing.py
index 250888f..7930452 100755
--- a/ruffus/test/test_pausing.py
+++ b/ruffus/test/test_pausing.py
@@ -8,93 +8,29 @@ from __future__ import print_function
"""
+import os
+import sys
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
-# options
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
-from optparse import OptionParser
-import sys, os
-import os.path
try:
- import StringIO as io
-except:
- import io as io
-
-import re
-
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-if __name__ == '__main__':
- module_name = os.path.split(sys.argv[0])[1]
- module_name = os.path.splitext(module_name)[0];
-else:
- module_name = __name__
-
-
-
-
-parser = OptionParser(version="%prog 1.0")
-parser.add_option("-D", "--debug", dest="debug",
- action="store_true", default=False,
- help="Make sure output is correct and clean up.")
-parser.add_option("-t", "--target_tasks", dest="target_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Target task(s) of pipeline.")
-parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Pipeline task(s) which will be included even if they are up to date.")
-parser.add_option("-j", "--jobs", dest="jobs",
- default=1,
- metavar="jobs",
- type="int",
- help="Specifies the number of jobs (commands) to run simultaneously.")
-parser.add_option("-v", "--verbose", dest = "verbose",
- action="count", default=0,
- help="Do not echo to shell but only print to log.")
-parser.add_option("-d", "--dependency", dest="dependency_file",
- #default="simple.svg",
- metavar="FILE",
- type="string",
- help="Print a dependency graph of the pipeline that would be executed "
- "to FILE, but do not execute it.")
-parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
- metavar="FORMAT",
- type="string",
- default = 'svg',
- help="format of dependency graph file. Can be 'ps' (PostScript), "+
- "'svg' 'svgz' (Structured Vector Graphics), " +
- "'png' 'gif' (bitmap graphics) etc ")
-parser.add_option("-n", "--just_print", dest="just_print",
- action="store_true", default=False,
- help="Print a description of the jobs that would be executed, "
- "but do not execute them.")
-parser.add_option("-M", "--minimal_rebuild_mode", dest="minimal_rebuild_mode",
- action="store_true", default=False,
- help="Rebuild a minimum of tasks necessary for the target. "
- "Ignore upstream out of date tasks if intervening tasks are fine.")
-parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
- action="store_true", default=False,
- help="Do not print out legend and key for dependency graph.")
-parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
- action="store_true", default=False,
- help="Draw horizontal dependency graph.")
-
-parameters = [
- ]
-
-
-
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
@@ -112,15 +48,14 @@ import sys,os
from collections import defaultdict
import random
-sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
+import json
# use simplejson in place of json for python < 2.6
-try:
- import json
-except ImportError:
- import simplejson
- json = simplejson
+#try:
+# import json
+#except ImportError:
+# import simplejson
+# json = simplejson
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -140,13 +75,14 @@ def test_job_io(infiles, outfiles, extra_params):
if isinstance(infiles, str):
infiles = [infiles]
- elif infiles == None:
+ elif infiles is None:
infiles = []
if isinstance(outfiles, str):
outfiles = [outfiles]
output_text = list()
for f in infiles:
- output_text.append(open(f).read())
+ with open(f) as ii:
+ output_text.append(ii.read())
output_text = "".join(sorted(output_text))
output_text += json.dumps(infiles) + " -> " + json.dumps(outfiles) + "\n"
for f in outfiles:
@@ -155,23 +91,6 @@ def test_job_io(infiles, outfiles, extra_params):
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# Main logic
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-
-
-
-
-# get help string
-f =io.StringIO()
-parser.print_help(f)
-helpstr = f.getvalue()
-(options, remaining_args) = parser.parse_args()
-
@@ -189,34 +108,42 @@ helpstr = f.getvalue()
#
tempdir = "test_pausing_dir/"
+def do_write(file_name, what):
+ with open(file_name, "a") as oo:
+ oo.write(what)
+test_file = tempdir + "task.done"
#
# task1
#
@files(None, [tempdir + d for d in ('a.1', 'b.1', 'c.1')])
@follows(mkdir(tempdir))
- at posttask(lambda: open(tempdir + "task.done", "a").write("Task 1 Done\n"))
+ at posttask(lambda: do_write(test_file, "Task 1 Done\n"))
def task1(infiles, outfiles, *extra_params):
"""
First task
"""
- open(tempdir + "jobs.start", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.start", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
test_job_io(infiles, outfiles, extra_params)
- open(tempdir + "jobs.finish", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.finish", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
#
# task2
#
- at posttask(lambda: open(tempdir + "task.done", "a").write("Task 2 Done\n"))
+ at posttask(lambda: do_write(test_file, "Task 2 Done\n"))
@follows(task1)
@transform(tempdir + "*.1", suffix(".1"), ".2")
def task2(infiles, outfiles, *extra_params):
"""
Second task
"""
- open(tempdir + "jobs.start", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.start", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
test_job_io(infiles, outfiles, extra_params)
- open(tempdir + "jobs.finish", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.finish", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
@@ -224,14 +151,16 @@ def task2(infiles, outfiles, *extra_params):
# task3
#
@transform(task2, regex('(.*).2'), inputs([r"\1.2", tempdir + "a.1"]), r'\1.3')
- at posttask(lambda: open(tempdir + "task.done", "a").write("Task 3 Done\n"))
+ at posttask(lambda: do_write(test_file, "Task 3 Done\n"))
def task3(infiles, outfiles, *extra_params):
"""
Third task
"""
- open(tempdir + "jobs.start", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.start", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
test_job_io(infiles, outfiles, extra_params)
- open(tempdir + "jobs.finish", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.finish", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
@@ -240,28 +169,32 @@ def task3(infiles, outfiles, *extra_params):
#
@transform(tempdir + "*.1", suffix(".1"), ".4")
@follows(task1)
- at posttask(lambda: open(tempdir + "task.done", "a").write("Task 4 Done\n"))
+ at posttask(lambda: do_write(test_file, "Task 4 Done\n"))
def task4(infiles, outfiles, *extra_params):
"""
Fourth task
"""
- open(tempdir + "jobs.start", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.start", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
test_job_io(infiles, outfiles, extra_params)
- open(tempdir + "jobs.finish", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.finish", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
#
# task5
#
@files(None, tempdir + 'a.5')
@follows(mkdir(tempdir))
- at posttask(lambda: open(tempdir + "task.done", "a").write("Task 5 Done\n"))
+ at posttask(lambda: do_write(test_file, "Task 5 Done\n"))
def task5(infiles, outfiles, *extra_params):
"""
Fifth task is extra slow
"""
- open(tempdir + "jobs.start", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.start", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
test_job_io(infiles, outfiles, extra_params)
- open(tempdir + "jobs.finish", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.finish", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
#
# task6
@@ -269,14 +202,16 @@ def task5(infiles, outfiles, *extra_params):
#@files([[[tempdir + d for d in 'a.3', 'b.3', 'c.3', 'a.4', 'b.4', 'c.4', 'a.5'], tempdir + 'final.6']])
@merge([task3, task4, task5], tempdir + "final.6")
@follows(task3, task4, task5, )
- at posttask(lambda: open(tempdir + "task.done", "a").write("Task 6 Done\n"))
+ at posttask(lambda: do_write(test_file, "Task 6 Done\n"))
def task6(infiles, outfiles, *extra_params):
"""
final task
"""
- open(tempdir + "jobs.start", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.start", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
test_job_io(infiles, outfiles, extra_params)
- open(tempdir + "jobs.finish", "a").write('job = %s\n' % json.dumps([infiles, outfiles]))
+ with open(tempdir + "jobs.finish", "a") as oo:
+ oo.write('job = %s\n' % json.dumps([infiles, outfiles]))
@@ -298,11 +233,12 @@ def check_job_order_correct(filename):
index_re = re.compile(r'.*\.([0-9])["\]\n]*$')
job_indices = defaultdict(list)
- for linenum, l in enumerate(open(filename)):
- m = index_re.search(l)
- if not m:
- raise "Non-matching line in [%s]" % filename
- job_indices[int(m.group(1))].append(linenum)
+ with open(filename) as ii:
+ for linenum, l in enumerate(ii):
+ m = index_re.search(l)
+ if not m:
+ raise "Non-matching line in [%s]" % filename
+ job_indices[int(m.group(1))].append(linenum)
for job_index in job_indices:
job_indices[job_index].sort()
@@ -339,7 +275,8 @@ def check_final_output_correct():
[] -> ["DIR/a.1", "DIR/b.1", "DIR/c.1"]
[] -> ["DIR/a.5"]"""
expected_output = expected_output.replace(" ", "").replace("DIR/", tempdir).split("\n")
- final_6_contents = sorted([l.rstrip() for l in open(tempdir + "final.6", "r").readlines()])
+ with open(tempdir + "final.6", "r") as ii:
+ final_6_contents = sorted([l.rstrip() for l in ii.readlines()])
if final_6_contents != expected_output:
for i, (l1, l2) in enumerate(zip(final_6_contents, expected_output)):
if l1 != l2:
@@ -347,40 +284,71 @@ def check_final_output_correct():
raise Exception ("Final.6 output is not as expected\n")
-#
-# Necessary to protect the "entry point" of the program under windows.
-# see: http://docs.python.org/library/multiprocessing.html#multiprocessing-programming
-#
-if __name__ == '__main__':
- if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
- long_winded=True,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
-
- elif options.dependency_file:
- pipeline_printout_graph ( open(options.dependency_file, "w"),
- options.dependency_graph_format,
- options.target_tasks,
- options.forced_tasks,
- draw_vertically = not options.draw_horizontally,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- no_key_legend = options.no_key_legend_in_graph)
- elif options.debug:
- import os
- os.system("rm -rf %s" % tempdir)
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- logger = stderr_logger if options.verbose else black_hole_logger,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- verbose = options.verbose)
+
+import unittest, shutil, os
+try:
+ from StringIO import StringIO
+except:
+ from io import StringIO
+
+class Test_ruffus(unittest.TestCase):
+ def setUp(self):
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+
+ def tearDown(self):
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+
+ def test_ruffus (self):
+ pipeline_run(multiprocess = 50, verbose = 0)
+ check_final_output_correct()
+ check_job_order_correct(tempdir + "jobs.start")
+ check_job_order_correct(tempdir + "jobs.finish")
+ def test_newstyle_ruffus (self):
+ test_pipeline = Pipeline("test")
+
+ test_pipeline.files(task1,
+ None,
+ [tempdir + d for d in ('a.1', 'b.1', 'c.1')])\
+ .follows(mkdir(tempdir))\
+ .posttask(lambda: do_write(test_file, "Task 1 Done\n"))
+
+ test_pipeline.transform(task2, tempdir + "*.1", suffix(".1"), ".2")\
+ .posttask(lambda: do_write(test_file, "Task 2 Done\n"))\
+ .follows(task1)
+
+ test_pipeline.transform(task3, task2, regex('(.*).2'), inputs([r"\1.2", tempdir + "a.1"]), r'\1.3')\
+ .posttask(lambda: do_write(test_file, "Task 3 Done\n"))
+
+ test_pipeline.transform(task_func = task4,
+ input = tempdir + "*.1",
+ filter = suffix(".1"),
+ output = ".4")\
+ .follows(task1)\
+ .posttask(lambda: do_write(test_file, "Task 4 Done\n"))
+
+ test_pipeline.files(task5, None, tempdir + 'a.5')\
+ .follows(mkdir(tempdir))\
+ .posttask(lambda: do_write(test_file, "Task 5 Done\n"))
+
+ test_pipeline.merge(task_func = task6,
+ input = [task3, task4, task5],
+ output = tempdir + "final.6")\
+ .follows(task3, task4, task5, )\
+ .posttask(lambda: do_write(test_file, "Task 6 Done\n"))
+
+ test_pipeline.run(multiprocess = 50, verbose = 0)
check_final_output_correct()
check_job_order_correct(tempdir + "jobs.start")
check_job_order_correct(tempdir + "jobs.finish")
- os.system("rm -rf %s" % tempdir)
- print("OK")
- else:
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- logger = stderr_logger if options.verbose else black_hole_logger,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- verbose = options.verbose)
+
+if __name__ == '__main__':
+ unittest.main()
+
diff --git a/ruffus/test/test_pipeline_printout_graph.py b/ruffus/test/test_pipeline_printout_graph.py
new file mode 100755
index 0000000..2f6b201
--- /dev/null
+++ b/ruffus/test/test_pipeline_printout_graph.py
@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+from __future__ import print_function
+"""
+
+ test_split_subdivide_checkpointing.py
+
+
+"""
+tempdir = "testing_dir/"
+
+import os
+import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+for attr in "pipeline_run", "pipeline_printout_graph", "originate", "split", "transform", "subdivide", "formatter", "Pipeline":
+ globals()[attr] = getattr (ruffus, attr)
+
+
+
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# imports
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+
+from subprocess import *
+
+
+
+import unittest
+import shutil
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# Each time the pipeline is FORCED to rerun,
+# More files are created for each task
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+
+
+
+ at originate([tempdir + 'start'])
+def make_start(outfile):
+ """
+ -> start
+ """
+ open(outfile, 'w').close()
+
+
+ at split(make_start, tempdir + '*.split')
+def split_start(infiles, outfiles):
+ """
+ -> XXX.split
+ where XXX = 0 .. N,
+ N = previous N + 1
+ """
+
+ # split always runs exactly one job (unlike @subdivide)
+ # So it implicitly combines all its inputs before running and generating multiple output
+ # @originate generates multiple output so the input for @split is a list...
+ infile = infiles[0]
+
+ # clean up previous
+ for f in outfiles:
+ os.unlink(f)
+
+
+ #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ #
+ # Create more files than the previous invocation
+ #
+ #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ n_to_produce = len(outfiles) + 1
+ for i in range(n_to_produce):
+ f = '{}{}.split'.format(tempdir, i)
+ open(f, 'a').close()
+
+
+
+ at subdivide(split_start, formatter(), tempdir + '{basename[0]}_*.subdivided', tempdir + '{basename[0]}')
+def subdivide_start(infile, outfiles, infile_basename):
+ # cleanup existing
+ for f in outfiles:
+ os.unlink(f)
+
+ #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ #
+ # Create more files than the previous invocation
+ #
+ #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ n_to_produce = len(outfiles) + 1
+ for i in range( n_to_produce):
+ open('{}_{}.subdivided'.format(infile_basename, i), 'a').close()
+
+
+
+class Test_ruffus(unittest.TestCase):
+
+ def tearDown(self):
+ # only tear down if not throw exception so we can debug?
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+
+ def setUp(self):
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+ os.makedirs(tempdir)
+
+ #
+ # check graphviz exists for turning dot files into jpg, svg etc
+ #
+ try:
+ process = Popen("echo what | dot", stdout=PIPE, stderr=STDOUT, shell = True)
+ output, unused_err = process.communicate()
+ retcode = process.poll()
+ if retcode:
+ raise CalledProcessError(retcode, "echo what | dot", output=output)
+ except CalledProcessError as err:
+ output_str = str(err.output)
+ if "No such file or directory" in output_str or "not found" in output_str or "Unable to access jarfile" in output_str:
+ self.graph_viz_present = False
+ return
+ self.graph_viz_present = True
+
+ def test_ruffus (self):
+
+ print(" Run pipeline normally...")
+ if self.graph_viz_present:
+ pipeline_printout_graph(tempdir + "flowchart.dot")
+ pipeline_printout_graph(tempdir + "flowchart.jpg",
+ target_tasks =[subdivide_start],
+ forcedtorun_tasks = [split_start],
+ no_key_legend = True)
+ pipeline_printout_graph(tempdir + "flowchart.svg", no_key_legend = False)
+ # Unknown format
+ try:
+ pipeline_printout_graph(tempdir + "flowchart.unknown", no_key_legend = False)
+ raise Exception("Failed to throw exception for pipeline_printout_graph unknown extension ")
+ except CalledProcessError as err:
+ pass
+ pipeline_printout_graph(tempdir + "flowchart.unknown", "svg", no_key_legend = False)
+
+ else:
+ pipeline_printout_graph(tempdir + "flowchart.dot",
+ target_tasks =[subdivide_start],
+ forcedtorun_tasks = [split_start],
+ no_key_legend = True)
+
+ def test_newstyle_ruffus (self):
+
+ print(" Run pipeline normally...")
+ test_pipeline = Pipeline("test")
+ test_pipeline.originate(make_start, [tempdir + 'start'])
+
+ test_pipeline.split(split_start, make_start, tempdir + '*.split')
+
+ test_pipeline.subdivide(subdivide_start, split_start, formatter(), tempdir + '{basename[0]}_*.subdivided', tempdir + '{basename[0]}')
+ if self.graph_viz_present:
+ test_pipeline.printout_graph(tempdir + "flowchart.dot")
+ test_pipeline.printout_graph(tempdir + "flowchart.jpg",
+ target_tasks =[subdivide_start],
+ forcedtorun_tasks = [split_start],
+ no_key_legend = True)
+ test_pipeline.printout_graph(tempdir + "flowchart.svg", no_key_legend = False)
+ # Unknown format
+ try:
+ test_pipeline.printout_graph(tempdir + "flowchart.unknown", no_key_legend = False)
+ raise Exception("Failed to throw exception for test_pipeline.printout_graph unknown extension ")
+ except CalledProcessError as err:
+ pass
+ test_pipeline.printout_graph(tempdir + "flowchart.unknown", "svg", no_key_legend = False)
+
+ else:
+ test_pipeline.printout_graph(tempdir + "flowchart.dot",
+ target_tasks =[subdivide_start],
+ forcedtorun_tasks = [split_start],
+ no_key_legend = True)
+
+
+
+if __name__ == '__main__':
+ unittest.main()
+
+
diff --git a/ruffus/test/test_posttask_merge.py b/ruffus/test/test_posttask_merge.py
new file mode 100755
index 0000000..ed5ccef
--- /dev/null
+++ b/ruffus/test/test_posttask_merge.py
@@ -0,0 +1,244 @@
+#!/usr/bin/env python
+from __future__ import print_function
+"""
+
+ test_files_post_merge.py
+
+ bug where @files follows merge and extra parenthesis inserted
+
+"""
+tempdir = "temp_filesre_split_and_combine/"
+test_file = tempdir + "test_output"
+jobs_per_task = 50
+
+import os
+import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+
+try:
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
+
+
+
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# imports
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+
+
+
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# Tasks
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+#
+# split_fasta_file
+#
+
+def do_write(file_name, what):
+ with open(file_name, "a") as oo:
+ oo.write(what)
+test_file = tempdir + "task.done"
+
+
+
+ at posttask(lambda: do_write(test_file, "Split into %d files\n" % jobs_per_task))
+ at split(tempdir + "original.fa", [tempdir + "files.split.success", tempdir + "files.split.*.fa"])
+def split_fasta_file (input_file, outputs):
+
+ #
+ # remove previous fasta files
+ #
+ success_flag = outputs[0]
+ output_file_names = outputs[1:]
+ for f in output_file_names:
+ os.unlink(f)
+
+ #
+ # create as many files as we are simulating in jobs_per_task
+ #
+ for i in range(jobs_per_task):
+ with open(tempdir + "files.split.%03d.fa" % i, "w") as oo:
+ pass
+
+ with open(success_flag, "w") as oo:
+ pass
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+#
+# align_sequences
+#
+ at posttask(lambda: do_write(test_file, "Sequences aligned\n"))
+ at transform(split_fasta_file, suffix(".fa"), ".aln") # fa -> aln
+def align_sequences (input_file, output_filename):
+ with open(output_filename, "w") as oo:
+ oo.write("%s\n" % output_filename)
+
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+#
+# percentage_identity
+#
+ at posttask(lambda: do_write(test_file, "%Identity calculated\n"))
+ at transform(align_sequences, # find all results from align_sequences
+ suffix(".aln"), # replace suffix with:
+ [r".pcid", # .pcid suffix for the result
+ r".pcid_success"]) # .pcid_success to indicate job completed
+def percentage_identity (input_file, output_files):
+ (output_filename, success_flag_filename) = output_files
+ with open(output_filename, "w") as oo:
+ oo.write("%s\n" % output_filename)
+ with open(success_flag_filename, "w") as oo:
+ pass
+
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+#
+# combine_results
+#
+ at posttask(lambda: do_write(test_file, "Results recombined\n"))
+ at merge(percentage_identity, tempdir + "all.combine_results")
+def combine_results (input_files, output_files):
+ """
+ Combine all
+ """
+ (output_filename) = output_files
+ with open(output_filename, "w") as oo:
+ for inp, flag in input_files:
+ with open(inp) as ii:
+ oo.write(ii.read())
+
+
+
+ at files(combine_results, os.path.join(tempdir, "check_all_is.well"))
+def post_merge_check (input_filename, output_filename):
+ """
+ check that merge sends just one file, not a list to me
+ """
+ with open(output_filename, "w") as oo:
+ with open(input_filename) as ii:
+ oo.write(ii.read())
+
+ at files(post_merge_check, os.path.join(tempdir, "check_all_is.weller"))
+def post_post_merge_check (input_filename, output_filename):
+ """
+ check that @files forwards a single file on when given a single file
+ """
+ with open(output_filename, "w") as oo:
+ with open(input_filename) as ii:
+ oo.write(ii.read())
+
+
+
+import unittest, shutil
+try:
+ from StringIO import StringIO
+except:
+ from io import StringIO
+
+class Test_ruffus(unittest.TestCase):
+ def setUp(self):
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+ os.makedirs(tempdir)
+ open(tempdir + "original.fa", "w").close()
+ self.expected_text = """Split into %d files
+Sequences aligned
+%%Identity calculated
+Results recombined
+""" % jobs_per_task
+
+
+ def tearDown(self):
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+
+ def test_ruffus (self):
+ pipeline_run(multiprocess = 50, verbose = 0)
+ with open(test_file) as ii:
+ post_task_text = ii.read()
+ self.assertEqual(post_task_text, self.expected_text)
+
+
+
+ def test_newstyle_ruffus (self):
+
+ test_pipeline = Pipeline("test")
+
+
+ test_pipeline.split( task_func = split_fasta_file,
+ input = tempdir + "original.fa",
+ output = [tempdir + "files.split.success",
+ tempdir + "files.split.*.fa"])\
+ .posttask(lambda: do_write(test_file, "Split into %d files\n" % jobs_per_task))
+
+
+ test_pipeline.transform(task_func = align_sequences,
+ input = split_fasta_file,
+ filter = suffix(".fa"),
+ output = ".aln" # fa -> aln
+ )\
+ .posttask(lambda: do_write(test_file, "Sequences aligned\n"))
+
+
+ test_pipeline.transform(task_func = percentage_identity,
+ input = align_sequences, # find all results from align_sequences
+ filter = suffix(".aln"), # replace suffix with:
+ output = [r".pcid", # .pcid suffix for the result
+ r".pcid_success"] # .pcid_success to indicate job completed
+ )\
+ .posttask(lambda: do_write(test_file, "%Identity calculated\n"))
+
+
+ test_pipeline.merge(task_func = combine_results,
+ input = percentage_identity,
+ output = tempdir + "all.combine_results")\
+ .posttask(lambda: do_write(test_file, "Results recombined\n"))
+
+ test_pipeline.files(post_merge_check, combine_results, os.path.join(tempdir, "check_all_is.well"))
+
+ test_pipeline.files(post_post_merge_check, post_merge_check, os.path.join(tempdir, "check_all_is.weller"))
+
+ test_pipeline.run(multiprocess = 50, verbose = 0)
+ with open(test_file) as ii:
+ post_task_text = ii.read()
+ self.assertEqual(post_task_text, self.expected_text)
+
+
+
+if __name__ == '__main__':
+ unittest.main()
+
diff --git a/ruffus/test/test_regex_error_messages.py b/ruffus/test/test_regex_error_messages.py
index 796aa7c..5db1c4b 100755
--- a/ruffus/test/test_regex_error_messages.py
+++ b/ruffus/test/test_regex_error_messages.py
@@ -27,30 +27,42 @@ from __future__ import print_function
SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
"""
+workdir = 'tmp_test_regex_error_messages'
+#sub-1s resolution in system?
+one_second_per_job = None
+parallelism = 2
+
+import os
+import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+for attr in "pipeline_run", "pipeline_printout", "suffix", "transform", "split", "merge", "dbdict", "follows", "originate", "Pipeline", "regex":
+ globals()[attr] = getattr (ruffus, attr)
+RUFFUS_HISTORY_FILE = ruffus.ruffus_utility.RUFFUS_HISTORY_FILE
+fatal_error_input_file_does_not_match = ruffus.ruffus_exceptions.fatal_error_input_file_does_not_match
+RethrownJobError = ruffus.ruffus_exceptions.RethrownJobError
+
+
+import re
import unittest
-import os, re
-import sys
import shutil
try:
from StringIO import StringIO
except:
from io import StringIO
-import time
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0, os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-from ruffus import (pipeline_run, pipeline_printout, suffix, transform, split,
- merge, dbdict, follows)
-from ruffus.ruffus_exceptions import *
-from ruffus.ruffus_utility import (RUFFUS_HISTORY_FILE)
-workdir = 'tmp_test_regex_error_messages'
-#sub-1s resolution in system?
-one_second_per_job = None
-parallelism = 2
#___________________________________________________________________________
#
# generate_initial_files1
@@ -284,7 +296,7 @@ class Test_regex_error_messages(unittest.TestCase):
s = StringIO()
pipeline_printout(s, [test_regex_task], verbose=5, wrap_width = 10000)
- self.assertTrue(re.search('Missing files\n\s+\[tmp_test_regex_error_messages/a_name.tmp1, tmp_test_regex_error_messages/a_name.tmp2', s.getvalue()))
+ self.assertTrue(re.search('Missing files.*\[tmp_test_regex_error_messages/a_name.tmp1, tmp_test_regex_error_messages/a_name.tmp2', s.getvalue(), re.DOTALL))
def test_regex_run(self):
@@ -320,7 +332,7 @@ class Test_regex_error_messages(unittest.TestCase):
s = StringIO()
pipeline_printout(s, [test_suffix_task], verbose=5, wrap_width = 10000)
- self.assertTrue(re.search('Missing files\n\s+\[tmp_test_regex_error_messages/a_name.tmp1, tmp_test_regex_error_messages/a_name.tmp2', s.getvalue()))
+ self.assertTrue(re.search('Missing files.*\[tmp_test_regex_error_messages/a_name.tmp1, tmp_test_regex_error_messages/a_name.tmp2', s.getvalue(), re.DOTALL))
def test_suffix_run(self):
"""Run transform(...,suffix()...)"""
@@ -436,9 +448,9 @@ if __name__ == '__main__':
#pipeline_printout(sys.stdout, [test_product_task], verbose = 3)
parallelism = 1
suite = unittest.TestLoader().loadTestsFromTestCase(Test_regex_error_messages)
- unittest.TextTestRunner(verbosity=2).run(suite)
+ unittest.TextTestRunner(verbosity=1).run(suite)
parallelism = 2
suite = unittest.TestLoader().loadTestsFromTestCase(Test_regex_error_messages)
- unittest.TextTestRunner(verbosity=2).run(suite)
+ unittest.TextTestRunner(verbosity=1).run(suite)
#unittest.main()
diff --git a/ruffus/test/test_ruffus_utility.py b/ruffus/test/test_ruffus_utility.py
index eac816f..7349c17 100755
--- a/ruffus/test/test_ruffus_utility.py
+++ b/ruffus/test/test_ruffus_utility.py
@@ -4,52 +4,45 @@ from __future__ import print_function
#
# test_ruffus_utility.py
#
-#
-# Copyright (c) 2009 Leo Goodstadt
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
#################################################################################
-
"""
test_ruffus_utility.py
"""
-# use simplejson in place of json for python < 2.6
-try:
- import json
-except ImportError:
- import simplejson
- json = simplejson
-import unittest, os,sys
-if __name__ != '__main__':
- raise Exception ("This is not a callable module [%s]" % __main__)
+import os
+import sys
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0, os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-from ruffus.ruffus_utility import *
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
-os.chdir(exe_path)
-import unittest, time
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+try:
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
+try:
+ attrlist = ruffus.ruffus_utility.__all__
+except AttributeError:
+ attrlist = dir (ruffus.ruffus_utility)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus.ruffus_utility, attr)
+
+
+import unittest
+
#_________________________________________________________________________________________
@@ -73,42 +66,42 @@ class Test_get_nested_tasks_or_globs(unittest.TestCase):
#
# test strings
#
- self.check_equal("test", (set(), set(), set()))
- self.check_equal([("test1",), "test2", 3], (set(), set(), set()))
+ self.check_equal("test", (list(), set(), set()))
+ self.check_equal([("test1",), "test2", 3], (list(), set(), set()))
#
# test missing
#
- self.check_equal((1,3, [5]), (set(), set(), set()))
- self.check_equal(None, (set(), set(), set()))
+ self.check_equal((1,3, [5]), (list(), set(), set()))
+ self.check_equal(None, (list(), set(), set()))
#
# test glob
#
- self.check_equal([("test1.*",), "test?2", 3], (set(), set(['test1.*', 'test?2']), set()))
+ self.check_equal([("test1.*",), "test?2", 3], (list(), set(['test1.*', 'test?2']), set()))
#
# test glob and string
#
- self.check_equal([("test*1",), (("test3",),),"test2", 3], (set(), set(['test*1']), set()))
+ self.check_equal([("test*1",), (("test3",),),"test2", 3], (list(), set(['test*1']), set()))
#
# test task function
#
- self.check_equal(is_glob, (set([is_glob]), set([]), set()))
- self.check_equal([is_glob, [1, "this", ["that*", 5]], [(get_strings_in_nested_sequence,)]], (
- set([is_glob, get_strings_in_nested_sequence]), set(["that*"]), set()))
+ self.check_equal(is_glob, (list([is_glob]), set([]), set()))
+ self.check_equal([is_glob, [1, "this", ["that*", 5]], [(get_strings_in_flattened_sequence,)]], (
+ [is_glob, get_strings_in_flattened_sequence], set(["that*"]), set()))
#
# test wrapper
#
- self.check_equal(output_from(is_glob, ["what", 7], 5), (set([is_glob, "what"]), set([]), set()))
+ self.check_equal(output_from(is_glob, ["what", 7], 5), ([is_glob, "what"], set([]), set()))
#_________________________________________________________________________________________
-# replace_func_names_with_tasks
+# replace_placeholders_with_tasks_in_input_params
#_________________________________________________________________________________________
-class Test_replace_func_names_with_tasks(unittest.TestCase):
+class Test_replace_placeholders_with_tasks_in_input_params(unittest.TestCase):
def setUp(self):
exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
os.chdir(exe_path)
@@ -118,10 +111,10 @@ class Test_replace_func_names_with_tasks(unittest.TestCase):
# self.assertRaises(ValueError, random.sample, self.seq, 20)
def check_equal (self, a,b, d):
- self.assertEqual(replace_func_names_with_tasks(a, d), b)
+ self.assertEqual(replace_placeholders_with_tasks_in_input_params(a, d), b)
- def test_replace_func_names_with_tasks(self):
- func_or_name_to_task = {is_glob: "FF is_glob", "what" : "FF what", get_strings_in_nested_sequence: "FF get_strings_in_nested_sequence"}
+ def test_replace_placeholders_with_tasks_in_input_params(self):
+ func_or_name_to_task = {is_glob: "FF is_glob", "what" : "FF what", get_strings_in_flattened_sequence: "FF get_strings_in_flattened_sequence"}
#
@@ -144,8 +137,8 @@ class Test_replace_func_names_with_tasks(unittest.TestCase):
# test task function
#
self.check_equal(is_glob, "FF is_glob", func_or_name_to_task)
- self.check_equal([is_glob, [1, "this", ["that*", 5]], [(get_strings_in_nested_sequence,)]],
- ["FF is_glob", [1, "this", ["that*", 5]], [("FF get_strings_in_nested_sequence",)]],
+ self.check_equal([is_glob, [1, "this", ["that*", 5]], [(get_strings_in_flattened_sequence,)]],
+ ["FF is_glob", [1, "this", ["that*", 5]], [("FF get_strings_in_flattened_sequence",)]],
func_or_name_to_task)
#
# test wrapper
@@ -200,14 +193,14 @@ class Test_non_str_sequence(unittest.TestCase):
#_________________________________________________________________________________________
-# get_strings_in_nested_sequence
+# get_strings_in_flattened_sequence
#_________________________________________________________________________________________
-class Test_get_strings_in_nested_sequence(unittest.TestCase):
+class Test_get_strings_in_flattened_sequence(unittest.TestCase):
- def test_get_strings_in_nested_sequence (self):
+ def test_get_strings_in_flattened_sequence (self):
"""
- get_strings_in_nested_sequence()
+ get_strings_in_flattened_sequence()
"""
class inherited_str (str):
#
@@ -221,12 +214,12 @@ class Test_get_strings_in_nested_sequence(unittest.TestCase):
def __init__ (self, *param):
list.__init__(self, *param)
- self.assertEqual(get_strings_in_nested_sequence("one"), ["one"])
- self.assertEqual(get_strings_in_nested_sequence(["one", "two"]), ["one", "two"])
- self.assertEqual(get_strings_in_nested_sequence(["one", 1, "two"]), ["one", "two"])
- self.assertEqual(get_strings_in_nested_sequence(["one", [1, ["two"]]]), ["one", "two"])
- self.assertEqual(get_strings_in_nested_sequence([inherited_str("one"), [1, ["two"]]]), [inherited_str("one"), "two"])
- self.assertEqual(get_strings_in_nested_sequence(inherited_list([inherited_str("one"), [1, ["two"]]])),
+ self.assertEqual(get_strings_in_flattened_sequence("one"), ["one"])
+ self.assertEqual(get_strings_in_flattened_sequence(["one", "two"]), ["one", "two"])
+ self.assertEqual(get_strings_in_flattened_sequence(["one", 1, "two"]), ["one", "two"])
+ self.assertEqual(get_strings_in_flattened_sequence(["one", [1, ["two"]]]), ["one", "two"])
+ self.assertEqual(get_strings_in_flattened_sequence([inherited_str("one"), [1, ["two"]]]), [inherited_str("one"), "two"])
+ self.assertEqual(get_strings_in_flattened_sequence(inherited_list([inherited_str("one"), [1, ["two"]]])),
inherited_list([inherited_str("one"), "two"]))
@@ -253,19 +246,19 @@ class Test_get_first_strings_in_nested_sequence(unittest.TestCase):
def __init__ (self, *param):
list.__init__(self, *param)
- self.assertEqual(get_strings_in_nested_sequence("one", True), ["one"])
- self.assertEqual(get_strings_in_nested_sequence(["one", "two"], True), ["one", "two"])
- self.assertEqual(get_strings_in_nested_sequence(["one", 1, "two"], True), ["one", "two"])
- self.assertEqual(get_strings_in_nested_sequence(["one", [1, ["two"]]], True), ["one", "two"])
- self.assertEqual(get_strings_in_nested_sequence([inherited_str("one"), [1, ["two"]]], True), [inherited_str("one"), "two"])
- self.assertEqual(get_strings_in_nested_sequence(inherited_list([inherited_str("one"), [1, ["two"]]]), True),
+ self.assertEqual(get_strings_in_flattened_sequence("one"), ["one"])
+ self.assertEqual(get_strings_in_flattened_sequence(["one", "two"]), ["one", "two"])
+ self.assertEqual(get_strings_in_flattened_sequence(["one", 1, "two"]), ["one", "two"])
+ self.assertEqual(get_strings_in_flattened_sequence(["one", [1, ["two"]]]), ["one", "two"])
+ self.assertEqual(get_strings_in_flattened_sequence([inherited_str("one"), [1, ["two"]]]), [inherited_str("one"), "two"])
+ self.assertEqual(get_strings_in_flattened_sequence(inherited_list([inherited_str("one"), [1, ["two"]]])),
inherited_list([inherited_str("one"), "two"]))
- self.assertEqual(get_strings_in_nested_sequence(["one", [1, ["two"], "three"]], True), ["one", "two"])
+ self.assertEqual(get_strings_in_flattened_sequence(["one", [1, ["two"], "three"]]), ["one", "two", "three"])
d = {"four" :4}
- self.assertEqual(get_strings_in_nested_sequence(["one", [1, [d, "two"], "three"]], True), ["one", "two"])
- self.assertEqual(get_strings_in_nested_sequence(None, True), [])
- self.assertEqual(get_strings_in_nested_sequence([], True), [])
- self.assertEqual(get_strings_in_nested_sequence([1,2,3, d], True), [])
+ self.assertEqual(get_strings_in_flattened_sequence(["one", [1, [d, "two"], "three"]]), ["one", "two", "three"])
+ self.assertEqual(get_strings_in_flattened_sequence(None), [])
+ self.assertEqual(get_strings_in_flattened_sequence([]), [])
+ self.assertEqual(get_strings_in_flattened_sequence([1,2,3, d]), [])
#_________________________________________________________________________________________
@@ -402,9 +395,9 @@ class Test_expand_nested_tasks_or_globs(unittest.TestCase):
def setUp(self):
exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
os.chdir(exe_path)
- t1 = task._task("module", "func1");
- t2 = task._task("module", "func2");
- t3 = task._task("module", "func3");
+ t1 = task.Task(print, "module.func1");
+ t2 = task.Task(print, "module.func2");
+ t3 = task.Task(print, "module.func3");
self.tasks = [t1, t2, t3]
# self.assertEqual(self.seq, range(10))
@@ -414,14 +407,14 @@ class Test_expand_nested_tasks_or_globs(unittest.TestCase):
def check_equal (self, a,b):
tasks, globs, runtime_data_names = get_nested_tasks_or_globs(a)
- func_or_name_to_task = dict(list(zip((non_str_sequence, get_strings_in_nested_sequence, "what"), self.tasks)))
+ func_or_name_to_task = dict(list(zip((non_str_sequence, get_strings_in_flattened_sequence, "what"), self.tasks)))
task_or_glob_to_files = dict()
#for f in func_or_name_to_task:
# print f, task_or_glob_to_files[func_or_name_to_task[f]]
task_or_glob_to_files[self.tasks[0] ] = ["t1a", "t1b"] # non_str_sequence
- task_or_glob_to_files[self.tasks[1] ] = ["t2"] # get_strings_in_nested_sequence
+ task_or_glob_to_files[self.tasks[1] ] = ["t2"] # get_strings_in_flattened_sequence
task_or_glob_to_files[self.tasks[2] ] = ["t3"] # "what"
task_or_glob_to_files["that*" ] = ["that1", "that2"]
task_or_glob_to_files["test*1" ] = ["test11","test21"]
@@ -429,7 +422,7 @@ class Test_expand_nested_tasks_or_globs(unittest.TestCase):
task_or_glob_to_files["test?2" ] = ["test12"]
- param_a = replace_func_names_with_tasks(a, func_or_name_to_task)
+ param_a = replace_placeholders_with_tasks_in_input_params(a, func_or_name_to_task)
self.assertEqual(expand_nested_tasks_or_globs(param_a, task_or_glob_to_files), b)
def test_expand_nested_tasks_or_globs(self):
@@ -463,9 +456,9 @@ class Test_expand_nested_tasks_or_globs(unittest.TestCase):
# test task function
#
self.check_equal(non_str_sequence, ["t1a", "t1b"])
- self.check_equal(get_strings_in_nested_sequence, ["t2"])
- self.check_equal([get_strings_in_nested_sequence, non_str_sequence], ["t2", "t1a", "t1b"])
- self.check_equal([non_str_sequence, [1, "this", ["that*", 5]], [(get_strings_in_nested_sequence,)]],
+ self.check_equal(get_strings_in_flattened_sequence, ["t2"])
+ self.check_equal([get_strings_in_flattened_sequence, non_str_sequence], ["t2", "t1a", "t1b"])
+ self.check_equal([non_str_sequence, [1, "this", ["that*", 5]], [(get_strings_in_flattened_sequence,)]],
['t1a', 't1b', [1, 'this', ['that1', 'that2', 5]], [('t2',)]])
#
# test wrapper
@@ -952,14 +945,9 @@ class Test_shorten_filenames_encoder (unittest.TestCase):
'[[.../ruffus/ruffus/test/something.py, .../ruffus/ruffus/test/something.py], '
'[.../ruffus/ruffus/test/something.py, .../ruffus/ruffus/test/something.py], 6]')
-#
-#
-# debug parameter ignored if called as a module
-#
-if sys.argv.count("--debug"):
- sys.argv.remove("--debug")
-#sys.argv.append("Test_regex_replace")
-unittest.main()
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/ruffus/test/test_ruffus_utility_parse_task_arguments.py b/ruffus/test/test_ruffus_utility_parse_task_arguments.py
new file mode 100755
index 0000000..8e83918
--- /dev/null
+++ b/ruffus/test/test_ruffus_utility_parse_task_arguments.py
@@ -0,0 +1,713 @@
+#!/usr/bin/env python
+from __future__ import print_function
+################################################################################
+#
+# test_ruffus_utility_parse_task_arguments.py
+#
+#################################################################################
+"""
+ test_ruffus_utility.py
+"""
+
+import unittest
+
+import os
+import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+try:
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
+try:
+ attrlist = ruffus.ruffus_utility.__all__
+except AttributeError:
+ attrlist = dir (ruffus.ruffus_utility)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus.ruffus_utility, attr)
+
+import unittest
+#_________________________________________________________________________________________
+
+# Test_parse_transform_args
+
+#_________________________________________________________________________________________
+class Test_parse_transform_args (unittest.TestCase):
+
+ def test_parse_transform_args(self):
+ expected_arguments = ["input", "filter", "modify_inputs", "output", "extras"]
+
+ empty_unnamed_arguments = []
+ empty_named_arguments = {}
+ orig_unnamed_arguments = ["*.txt", suffix(".txt"), ".result", 1,2,3,4]
+ task_description = "@transform(%s)\ndef myfunc(...)\n"
+ expected_results = { 'input' : orig_unnamed_arguments[0],
+ 'filter' : orig_unnamed_arguments[1],
+ 'output' : orig_unnamed_arguments[2],
+ 'extras' : orig_unnamed_arguments[3:],
+ 'modify_inputs_mode': 2,
+ 'modify_inputs': None}
+ add_inputs_expected_results = { 'input' : orig_unnamed_arguments[0],
+ 'filter' : orig_unnamed_arguments[1],
+ 'output' : orig_unnamed_arguments[2],
+ 'extras' : orig_unnamed_arguments[3:],
+ 'modify_inputs_mode': 0,
+ 'modify_inputs': ("a.test", "b.test")}
+ replace_inputs_expected_results = { 'input' : orig_unnamed_arguments[0],
+ 'filter' : orig_unnamed_arguments[1],
+ 'output' : orig_unnamed_arguments[2],
+ 'extras' : orig_unnamed_arguments[3:],
+ 'modify_inputs_mode': 1,
+ 'modify_inputs': ("a.test", "b.test")}
+
+ # Error: empty list
+ with self.assertRaises(error_missing_args):
+ parse_task_arguments (empty_named_arguments, empty_named_arguments, expected_arguments, task_description)
+
+ # parse complete correctly
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {}, expected_arguments, task_description)
+ self.assertEqual(results, expected_results)
+
+ # Error: missing argument
+ unnamed_arguments = orig_unnamed_arguments[0:1]
+ with self.assertRaises(error_missing_args):
+ results = parse_task_arguments (unnamed_arguments,
+ {}, expected_arguments, task_description)
+
+ # parse almost complete and rescued with named parameter
+ unnamed_arguments = orig_unnamed_arguments[0:2]
+ results = parse_task_arguments (unnamed_arguments,
+ {'output' : orig_unnamed_arguments[2],
+ 'extras' : orig_unnamed_arguments[3:]},
+ expected_arguments, task_description)
+ self.assertEqual(results, expected_results)
+
+ # All named parameters
+ results = parse_task_arguments ([],
+ {'input' : orig_unnamed_arguments[0],
+ 'filter' : orig_unnamed_arguments[1],
+ 'output' : orig_unnamed_arguments[2],
+ 'extras' : orig_unnamed_arguments[3:]
+ },
+ expected_arguments, task_description)
+ self.assertEqual(results, expected_results)
+
+ # filter not regex suffix or formatter
+ with self.assertRaises(TypeError):
+ results = parse_task_arguments ([],
+ {'input' : orig_unnamed_arguments[0],
+ 'filter' : "a",
+ 'output' : orig_unnamed_arguments[2],
+ 'extras' : orig_unnamed_arguments[3:]
+ },
+ expected_arguments, task_description)
+
+ # Error: Unknown named arguments
+ with self.assertRaises(error_too_many_args):
+ results = parse_task_arguments ([],
+ {'input' : orig_unnamed_arguments[0],
+ 'filter' : orig_unnamed_arguments[1],
+ 'output' : orig_unnamed_arguments[2],
+ 'what' : orig_unnamed_arguments[3:]
+ },
+ expected_arguments, task_description)
+
+ # Error: Duplicate named arguments
+ with self.assertRaises(error_too_many_args):
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {'input' : orig_unnamed_arguments[0],
+ 'extras' : orig_unnamed_arguments[3:]
+ },
+ expected_arguments, task_description)
+
+
+ # add_inputs correct via named
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"add_inputs" : ("a.test", "b.test")}, expected_arguments, task_description)
+ self.assertEqual(results, add_inputs_expected_results)
+
+ # add_inputs correct via named and paranoid add_inputs wrapping
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"add_inputs" : add_inputs("a.test", "b.test")}, expected_arguments, task_description)
+ self.assertEqual(results, add_inputs_expected_results)
+
+ # add_inputs correct via unnamed
+ unnamed_arguments = list(orig_unnamed_arguments)
+ unnamed_arguments.insert(2, add_inputs("a.test", "b.test"))
+ results = parse_task_arguments (unnamed_arguments, {},
+ expected_arguments, task_description)
+ self.assertEqual(results, add_inputs_expected_results)
+
+ # replace_inputs correct via named
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"replace_inputs" : ("a.test", "b.test")}, expected_arguments, task_description)
+ self.assertEqual(results, replace_inputs_expected_results)
+
+ # replace_inputs correct via named and paranoid inputs() wrapping
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"replace_inputs" : inputs(("a.test", "b.test"))}, expected_arguments, task_description)
+ self.assertEqual(results, replace_inputs_expected_results)
+
+
+ # replace_inputs correct via unnamed
+ unnamed_arguments = list(orig_unnamed_arguments)
+ unnamed_arguments.insert(2, inputs(("a.test", "b.test")))
+ results = parse_task_arguments (unnamed_arguments, {},
+ expected_arguments, task_description)
+ self.assertEqual(results, replace_inputs_expected_results)
+
+
+
+ # Error: both add_inputs and replace_inputs via named
+ with self.assertRaises(error_too_many_args):
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"replace_inputs" : ("a.test", "b.test"),
+ "add_inputs" : ("a.test", "b.test")}, expected_arguments, task_description)
+
+ # Error: both add_inputs and replace_inputs via named / unnamed
+ with self.assertRaises(error_too_many_args):
+ unnamed_arguments = list(orig_unnamed_arguments)
+ unnamed_arguments.insert(2, inputs(("a.test", "b.test")))
+ results = parse_task_arguments (unnamed_arguments, {"add_inputs" : ("a.test", "b.test")},
+ expected_arguments, task_description)
+
+ # Error: both add_inputs and replace_inputs via named / unnamed
+ with self.assertRaises(error_too_many_args):
+ unnamed_arguments = list(orig_unnamed_arguments)
+ unnamed_arguments.insert(2, add_inputs("a.test", "b.test"))
+ results = parse_task_arguments (unnamed_arguments, {"replace_inputs" : ("a.test", "b.test")},
+ expected_arguments, task_description)
+
+ # Error: wrong number of arguments
+ with self.assertRaises(error_inputs_multiple_args):
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"replace_inputs" : inputs("a.test", "b.test")}, expected_arguments, task_description)
+
+ with self.assertRaises(error_inputs_multiple_args):
+ unnamed_arguments = orig_unnamed_arguments[0:2] + [inputs("a.test", "b.test")] + orig_unnamed_arguments[2:]
+ results = parse_task_arguments (unnamed_arguments, {}, expected_arguments, task_description)
+
+ # Error: no arguments
+ with self.assertRaises(error_inputs_multiple_args):
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"replace_inputs" : inputs()}, expected_arguments, task_description)
+
+
+#________________________________________________________________________________________________________
+
+# Test_parse_product_args
+
+#________________________________________________________________________________________________________
+class Test_parse_product_args (unittest.TestCase):
+ """
+ Make sure (input, filter, input2, filter2, input3, filter3,..., output, extras...) works
+ for @product
+ """
+
+ def test_parse_product_args(self):
+ self.maxDiff = None
+ expected_arguments = ["input", "filter", "inputN", "modify_inputs", "output", "extras"]
+
+ empty_unnamed_arguments = []
+ empty_named_arguments = {}
+ orig_unnamed_arguments = ["*.txt", formatter(".txt"), "*.contig", formatter(), "*.genome", formatter(), "{basename[0][0]}_{basename[1][0]}.result", 1,2,3,4]
+ task_description = "@product(%s)\ndef myfunc(...)\n"
+ expected_results = { 'input' : [orig_unnamed_arguments[0], orig_unnamed_arguments[2], orig_unnamed_arguments[4]],
+ 'filter' : [orig_unnamed_arguments[1], orig_unnamed_arguments[3], orig_unnamed_arguments[5]],
+ 'output' : orig_unnamed_arguments[6],
+ 'extras' : orig_unnamed_arguments[7:],
+ 'modify_inputs_mode': 2,
+ 'modify_inputs': None}
+ add_inputs_expected_results = { 'input' : [orig_unnamed_arguments[0], orig_unnamed_arguments[2], orig_unnamed_arguments[4]],
+ 'filter' : [orig_unnamed_arguments[1], orig_unnamed_arguments[3], orig_unnamed_arguments[5]],
+ 'output' : orig_unnamed_arguments[6],
+ 'extras' : orig_unnamed_arguments[7:],
+ 'modify_inputs_mode': 0,
+ 'modify_inputs': ("a.test", "b.test")}
+ replace_inputs_expected_results = { 'input' : [orig_unnamed_arguments[0], orig_unnamed_arguments[2], orig_unnamed_arguments[4]],
+ 'filter' : [orig_unnamed_arguments[1], orig_unnamed_arguments[3], orig_unnamed_arguments[5]],
+ 'output' : orig_unnamed_arguments[6],
+ 'extras' : orig_unnamed_arguments[7:],
+ 'modify_inputs_mode': 1,
+ 'modify_inputs': ("a.test", "b.test")}
+
+ # Error: empty list
+ with self.assertRaises(error_missing_args):
+ parse_task_arguments (empty_named_arguments, empty_named_arguments, expected_arguments, task_description)
+
+ # parse complete correctly
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {}, expected_arguments, task_description)
+ self.assertEqual(results, expected_results)
+
+ # Error: missing argument
+ unnamed_arguments = orig_unnamed_arguments[0:6]
+ with self.assertRaises(error_missing_args):
+ results = parse_task_arguments (unnamed_arguments,
+ {}, expected_arguments, task_description)
+
+ # parse almost complete and rescued with named parameter
+ unnamed_arguments = orig_unnamed_arguments[0:6]
+ results = parse_task_arguments (unnamed_arguments,
+ {'output' : expected_results['output'],
+ 'extras' : expected_results['extras']},
+ expected_arguments, task_description)
+ self.assertEqual(results, expected_results)
+
+ # All named parameters
+ results = parse_task_arguments ([],
+ {'input' : orig_unnamed_arguments[0],
+ 'filter' : orig_unnamed_arguments[1],
+ 'input2' : orig_unnamed_arguments[2],
+ 'filter2' : orig_unnamed_arguments[3],
+ 'input3' : orig_unnamed_arguments[4],
+ 'filter3' : orig_unnamed_arguments[5],
+ 'output' : orig_unnamed_arguments[6],
+ 'extras' : orig_unnamed_arguments[7:]
+ },
+ expected_arguments, task_description)
+ self.assertEqual(results, expected_results)
+
+
+ # Error: Unknown named arguments
+ with self.assertRaises(error_too_many_args):
+ results = parse_task_arguments ([],
+ {'input' : orig_unnamed_arguments[0],
+ 'filter' : orig_unnamed_arguments[1],
+ 'input2' : orig_unnamed_arguments[2],
+ 'filter2' : orig_unnamed_arguments[3],
+ 'input3' : orig_unnamed_arguments[4],
+ 'filter3' : orig_unnamed_arguments[5],
+ 'output' : orig_unnamed_arguments[6],
+ 'what' : orig_unnamed_arguments[7:]
+ },
+ expected_arguments, task_description)
+
+ # Error: Duplicate named arguments
+ with self.assertRaises(error_too_many_args):
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {'input' : orig_unnamed_arguments[0],
+ 'extras' : orig_unnamed_arguments[7:]
+ },
+ expected_arguments, task_description)
+
+
+ # add_inputs correct via named
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"add_inputs" : ("a.test", "b.test")}, expected_arguments, task_description)
+ self.assertEqual(results, add_inputs_expected_results)
+
+ # add_inputs correct via named and paranoid add_inputs wrapping
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"add_inputs" : add_inputs("a.test", "b.test")}, expected_arguments, task_description)
+ self.assertEqual(results, add_inputs_expected_results)
+
+ # add_inputs correct via unnamed
+ unnamed_arguments = list(orig_unnamed_arguments)
+ unnamed_arguments.insert(6, add_inputs("a.test", "b.test"))
+ results = parse_task_arguments (unnamed_arguments, {},
+ expected_arguments, task_description)
+ self.assertEqual(results, add_inputs_expected_results)
+
+ # replace_inputs correct via named
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"replace_inputs" : ("a.test", "b.test")}, expected_arguments, task_description)
+ self.assertEqual(results, replace_inputs_expected_results)
+
+ # replace_inputs correct via named and paranoid inputs() wrapping
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"replace_inputs" : inputs(("a.test", "b.test"))}, expected_arguments, task_description)
+ self.assertEqual(results, replace_inputs_expected_results)
+
+
+ # replace_inputs correct via unnamed
+ unnamed_arguments = list(orig_unnamed_arguments)
+ unnamed_arguments.insert(6, inputs(("a.test", "b.test")))
+ results = parse_task_arguments (unnamed_arguments, {},
+ expected_arguments, task_description)
+ self.assertEqual(results, replace_inputs_expected_results)
+
+
+
+ # Error: both add_inputs and replace_inputs via named
+ with self.assertRaises(error_too_many_args):
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"replace_inputs" : ("a.test", "b.test"),
+ "add_inputs" : ("a.test", "b.test")}, expected_arguments, task_description)
+
+ # Error: both add_inputs and replace_inputs via named / unnamed
+ with self.assertRaises(error_too_many_args):
+ unnamed_arguments = list(orig_unnamed_arguments)
+ unnamed_arguments.insert(6, inputs(("a.test", "b.test")))
+ results = parse_task_arguments (unnamed_arguments, {"add_inputs" : ("a.test", "b.test")},
+ expected_arguments, task_description)
+
+ # Error: both add_inputs and replace_inputs via named / unnamed
+ with self.assertRaises(error_too_many_args):
+ unnamed_arguments = list(orig_unnamed_arguments)
+ unnamed_arguments.insert(6, add_inputs("a.test", "b.test"))
+ results = parse_task_arguments (unnamed_arguments, {"replace_inputs" : ("a.test", "b.test")},
+ expected_arguments, task_description)
+
+
+ # Error: wrong number of arguments
+ with self.assertRaises(error_inputs_multiple_args):
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"replace_inputs" : inputs("a.test", "b.test")}, expected_arguments, task_description)
+
+ # Error: no arguments
+ with self.assertRaises(error_inputs_multiple_args):
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"replace_inputs" : inputs()}, expected_arguments, task_description)
+
+
+#________________________________________________________________________________________________________
+
+# Test_parse_combinatorics_args
+
+#________________________________________________________________________________________________________
+class Test_parse_combinatorics_args (unittest.TestCase):
+ """
+ Make sure (input, filter, tuple_size, output, extras...) works
+ for @combinations
+ """
+
+ def test_parse_combinations_args(self):
+ expected_arguments = ["input", "filter", "tuple_size", "modify_inputs", "output", "extras"]
+
+ empty_unnamed_arguments = []
+ empty_named_arguments = {}
+ orig_unnamed_arguments = ["*.txt", suffix(".txt"), 5, ".result", 1,2,3,4]
+ task_description = "@combinations(%s)\ndef myfunc(...)\n"
+ expected_results = { 'input' : orig_unnamed_arguments[0],
+ 'filter' : orig_unnamed_arguments[1],
+ 'tuple_size': orig_unnamed_arguments[2],
+ 'output' : orig_unnamed_arguments[3],
+ 'extras' : orig_unnamed_arguments[4:],
+ 'modify_inputs_mode': 2,
+ 'modify_inputs': None}
+ add_inputs_expected_results = { 'input' : orig_unnamed_arguments[0],
+ 'filter' : orig_unnamed_arguments[1],
+ 'tuple_size': orig_unnamed_arguments[2],
+ 'output' : orig_unnamed_arguments[3],
+ 'extras' : orig_unnamed_arguments[4:],
+ 'modify_inputs_mode': 0,
+ 'modify_inputs': ("a.test", "b.test")}
+ replace_inputs_expected_results = { 'input' : orig_unnamed_arguments[0],
+ 'filter' : orig_unnamed_arguments[1],
+ 'tuple_size': orig_unnamed_arguments[2],
+ 'output' : orig_unnamed_arguments[3],
+ 'extras' : orig_unnamed_arguments[4:],
+ 'modify_inputs_mode': 1,
+ 'modify_inputs': ("a.test", "b.test")}
+
+ # Error: empty list
+ with self.assertRaises(error_missing_args):
+ parse_task_arguments (empty_named_arguments, empty_named_arguments, expected_arguments, task_description)
+
+ # parse complete correctly
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {}, expected_arguments, task_description)
+ self.assertEqual(results, expected_results)
+
+ # Error tuple_size not int
+ unnamed_arguments = orig_unnamed_arguments[:]
+ unnamed_arguments[2] = 'a'
+ with self.assertRaises(TypeError):
+ results = parse_task_arguments (unnamed_arguments,
+ {}, expected_arguments, task_description)
+
+
+ # Error: missing argument
+ unnamed_arguments = orig_unnamed_arguments[0:2]
+ with self.assertRaises(error_missing_args):
+ results = parse_task_arguments (unnamed_arguments,
+ {}, expected_arguments, task_description)
+
+ # parse almost complete and rescued with named parameter
+ unnamed_arguments = orig_unnamed_arguments[0:2]
+ results = parse_task_arguments (unnamed_arguments,
+ {
+ 'tuple_size': orig_unnamed_arguments[2],
+ 'output' : orig_unnamed_arguments[3],
+ 'extras' : orig_unnamed_arguments[4:]},
+ expected_arguments, task_description)
+ self.assertEqual(results, expected_results)
+
+ # All named parameters
+ unnamed_arguments = orig_unnamed_arguments[0:2]
+ results = parse_task_arguments ([],
+ {'input' : orig_unnamed_arguments[0],
+ 'filter' : orig_unnamed_arguments[1],
+ 'tuple_size': orig_unnamed_arguments[2],
+ 'output' : orig_unnamed_arguments[3],
+ 'extras' : orig_unnamed_arguments[4:]
+ },
+ expected_arguments, task_description)
+ self.assertEqual(results, expected_results)
+
+
+ # Error tuple_size not int
+ unnamed_arguments = orig_unnamed_arguments[0:2]
+ with self.assertRaises(TypeError):
+ results = parse_task_arguments ([],
+ {'input' : orig_unnamed_arguments[0],
+ 'filter' : orig_unnamed_arguments[1],
+ 'tuple_size': "a",
+ 'output' : orig_unnamed_arguments[3],
+ 'extras' : orig_unnamed_arguments[4:]
+ },
+ expected_arguments, task_description)
+
+
+ # Error: Unknown named arguments
+ with self.assertRaises(error_too_many_args):
+ results = parse_task_arguments ([],
+ {'input' : orig_unnamed_arguments[0],
+ 'filter' : orig_unnamed_arguments[1],
+ 'tuple_size': orig_unnamed_arguments[2],
+ 'output' : orig_unnamed_arguments[3],
+ 'what' : orig_unnamed_arguments[4:]
+ },
+ expected_arguments, task_description)
+
+ # Error: Duplicate named arguments
+ with self.assertRaises(error_too_many_args):
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {'input' : orig_unnamed_arguments[0],
+ 'extras' : orig_unnamed_arguments[3:]
+ },
+ expected_arguments, task_description)
+
+
+ # add_inputs correct via named
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"add_inputs" : ("a.test", "b.test")}, expected_arguments, task_description)
+ self.assertEqual(results, add_inputs_expected_results)
+
+ # add_inputs correct via named and paranoid add_inputs wrapping
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"add_inputs" : add_inputs("a.test", "b.test")}, expected_arguments, task_description)
+ self.assertEqual(results, add_inputs_expected_results)
+
+ # add_inputs correct via unnamed
+ unnamed_arguments = list(orig_unnamed_arguments)
+ unnamed_arguments.insert(3, add_inputs("a.test", "b.test"))
+ results = parse_task_arguments (unnamed_arguments, {},
+ expected_arguments, task_description)
+ self.assertEqual(results, add_inputs_expected_results)
+
+ # replace_inputs correct via named
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"replace_inputs" : ("a.test", "b.test")}, expected_arguments, task_description)
+ self.assertEqual(results, replace_inputs_expected_results)
+
+ # replace_inputs correct via named and paranoid inputs() wrapping
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"replace_inputs" : inputs(("a.test", "b.test"))}, expected_arguments, task_description)
+ self.assertEqual(results, replace_inputs_expected_results)
+
+
+ # replace_inputs correct via unnamed
+ unnamed_arguments = list(orig_unnamed_arguments)
+ unnamed_arguments.insert(3, inputs(("a.test", "b.test")))
+ results = parse_task_arguments (unnamed_arguments, {},
+ expected_arguments, task_description)
+ self.assertEqual(results, replace_inputs_expected_results)
+
+
+
+ # Error: both add_inputs and replace_inputs via named
+ with self.assertRaises(error_too_many_args):
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"replace_inputs" : ("a.test", "b.test"),
+ "add_inputs" : ("a.test", "b.test")}, expected_arguments, task_description)
+
+ # Error: both add_inputs and replace_inputs via named / unnamed
+ with self.assertRaises(error_too_many_args):
+ unnamed_arguments = list(orig_unnamed_arguments)
+ unnamed_arguments.insert(3, inputs(("a.test", "b.test")))
+ results = parse_task_arguments (unnamed_arguments, {"add_inputs" : ("a.test", "b.test")},
+ expected_arguments, task_description)
+
+ # Error: both add_inputs and replace_inputs via named / unnamed
+ with self.assertRaises(error_too_many_args):
+ unnamed_arguments = list(orig_unnamed_arguments)
+ unnamed_arguments.insert(3, add_inputs("a.test", "b.test"))
+ results = parse_task_arguments (unnamed_arguments, {"replace_inputs" : ("a.test", "b.test")},
+ expected_arguments, task_description)
+
+ # Error: wrong number of arguments
+ with self.assertRaises(error_inputs_multiple_args):
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"replace_inputs" : inputs("a.test", "b.test")}, expected_arguments, task_description)
+
+ # Error: no arguments
+ with self.assertRaises(error_inputs_multiple_args):
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {"replace_inputs" : inputs()}, expected_arguments, task_description)
+
+
+#________________________________________________________________________________________________________
+
+# Test_parse_originate_args
+
+#________________________________________________________________________________________________________
+class Test_parse_originate_args (unittest.TestCase):
+ """
+ Make sure @originate(output, extras...) works
+ """
+
+ def test_parse_originate_args(self):
+ expected_arguments = ["output", "extras"]
+
+ empty_unnamed_arguments = []
+ empty_named_arguments = {}
+ orig_unnamed_arguments = [["a.1","b.1"], 1,2,3,4]
+ task_description = "@originate(%s)\ndef myfunc(...)\n"
+ expected_results = { 'output' : orig_unnamed_arguments[0],
+ 'extras' : orig_unnamed_arguments[1:]}
+
+ # Error: empty list
+ with self.assertRaises(error_missing_args):
+ parse_task_arguments (empty_named_arguments, empty_named_arguments, expected_arguments, task_description)
+
+ # parse complete correctly
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {}, expected_arguments, task_description)
+ self.assertEqual(results, expected_results)
+
+ # All named parameters
+ unnamed_arguments = orig_unnamed_arguments[0:2]
+ results = parse_task_arguments ([],
+ {'output' : orig_unnamed_arguments[0],
+ 'extras' : orig_unnamed_arguments[1:]
+ },
+ expected_arguments, task_description)
+ self.assertEqual(results, expected_results)
+
+
+ # Error: Unknown named arguments
+ with self.assertRaises(error_too_many_args):
+ results = parse_task_arguments ([],
+ {'output' : orig_unnamed_arguments[0],
+ 'what' : orig_unnamed_arguments[1:]
+ },
+ expected_arguments, task_description)
+
+ # Error: Duplicate named arguments
+ with self.assertRaises(error_too_many_args):
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {'output' : orig_unnamed_arguments[0],
+ },
+ expected_arguments, task_description)
+
+#_________________________________________________________________________________________
+
+# Test_parse_mkdir_args
+
+#_________________________________________________________________________________________
+class Test_parse_mkdir_args (unittest.TestCase):
+
+ def test_parse_mkdir_args(self):
+ expected_arguments = ["input", "filter", "output"]
+
+ empty_unnamed_arguments = []
+ empty_named_arguments = {}
+ orig_unnamed_arguments = ["*.txt", suffix(".txt"), ".result"]
+ task_description = "@mkdir(%s)\ndef myfunc(...)\n"
+ expected_results = { 'input' : orig_unnamed_arguments[0],
+ 'filter' : orig_unnamed_arguments[1],
+ 'output' : orig_unnamed_arguments[2]}
+ add_inputs_expected_results = { 'input' : orig_unnamed_arguments[0],
+ 'filter' : orig_unnamed_arguments[1],
+ 'output' : orig_unnamed_arguments[2]}
+ replace_inputs_expected_results = { 'input' : orig_unnamed_arguments[0],
+ 'filter' : orig_unnamed_arguments[1],
+ 'output' : orig_unnamed_arguments[2]}
+
+ # Error: empty list
+ with self.assertRaises(error_missing_args):
+ parse_task_arguments (empty_named_arguments, empty_named_arguments, expected_arguments, task_description)
+
+ # parse complete correctly
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {}, expected_arguments, task_description)
+ self.assertEqual(results, expected_results)
+
+ # Error: missing argument
+ unnamed_arguments = orig_unnamed_arguments[0:1]
+ with self.assertRaises(error_missing_args):
+ results = parse_task_arguments (unnamed_arguments,
+ {}, expected_arguments, task_description)
+
+ # parse almost complete and rescued with named parameter
+ unnamed_arguments = orig_unnamed_arguments[0:2]
+ results = parse_task_arguments (unnamed_arguments,
+ {'output' : orig_unnamed_arguments[2]},
+ expected_arguments, task_description)
+ self.assertEqual(results, expected_results)
+
+ # All named parameters
+ unnamed_arguments = orig_unnamed_arguments[0:2]
+ results = parse_task_arguments ([],
+ {'input' : orig_unnamed_arguments[0],
+ 'filter' : orig_unnamed_arguments[1],
+ 'output' : orig_unnamed_arguments[2],
+ },
+ expected_arguments, task_description)
+ self.assertEqual(results, expected_results)
+
+
+ # Error: Unknown named arguments
+ with self.assertRaises(error_too_many_args):
+ results = parse_task_arguments ([],
+ {'input' : orig_unnamed_arguments[0],
+ 'filter' : orig_unnamed_arguments[1],
+ 'output' : orig_unnamed_arguments[2],
+ 'what' : orig_unnamed_arguments[3:]
+ },
+ expected_arguments, task_description)
+
+ # Error: Duplicate named arguments
+ with self.assertRaises(error_too_many_args):
+ results = parse_task_arguments (orig_unnamed_arguments,
+ {'input' : orig_unnamed_arguments[0],
+ },
+ expected_arguments, task_description)
+
+ # Error: no extras arguments allowed
+ with self.assertRaises(error_too_many_args):
+ results = parse_task_arguments (orig_unnamed_arguments + [1,formatter(),'a',4],
+ {'input' : orig_unnamed_arguments[0],
+ },
+ expected_arguments, task_description)
+
+
+
+
+#
+# Necessary to protect the "entry point" of the program under windows.
+# see: http://docs.python.org/library/multiprocessing.html#multiprocessing-programming
+#
+if __name__ == '__main__':
+ unittest.main()
+
+
+
diff --git a/ruffus/test/test_runtime_data.py b/ruffus/test/test_runtime_data.py
new file mode 100755
index 0000000..e958dc9
--- /dev/null
+++ b/ruffus/test/test_runtime_data.py
@@ -0,0 +1,187 @@
+#!/usr/bin/env python
+from __future__ import print_function
+"""
+
+ test_tasks.py
+
+"""
+
+runtime_files = ["a.3"]
+
+import os
+import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+
+try:
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
+
+
+
+
+
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# imports
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+import unittest
+import shutil
+
+import json
+# use simplejson in place of json for python < 2.6
+#try:
+# import json
+#except ImportError:
+# import simplejson
+# json = simplejson
+
+
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# Tasks
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+
+#
+# task1
+#
+ at originate(['a.1'] + runtime_files)
+def task1(outfile):
+ """
+ First task
+ """
+ output_text = ""
+ output_text += " -> " + json.dumps(outfile) + "\n"
+ with open(outfile, "w") as oo:
+ oo.write(output_text)
+
+
+
+#
+# task2
+#
+ at transform(task1, suffix(".1"), ".2")
+def task2(infile, outfile):
+ """
+ Second task
+ """
+ if infile:
+ with open(infile) as ii:
+ output_text = ii.read()
+ else:
+ output_text = ""
+ output_text += json.dumps(infile) + " -> " + json.dumps(outfile) + "\n"
+ with open(outfile, "w") as oo:
+ oo.write(output_text)
+
+
+
+#
+# task3
+#
+ at transform(task2, suffix(".2"), ".3")
+def task3(infile, outfile):
+ """
+ Third task
+ """
+ if infile:
+ with open(infile) as ii:
+ output_text = ii.read()
+ else:
+ output_text = ""
+ output_text += json.dumps(infile) + " -> " + json.dumps(outfile) + "\n"
+ with open(outfile, "w") as oo:
+ oo.write(output_text)
+
+
+
+#
+# task4
+#
+ at follows(task3)
+ at transform(runtime_parameter("a"), suffix(".3"), ".4")
+def task4(infile, outfile):
+ """
+ Fourth task
+ """
+ if infile:
+ with open(infile) as ii:
+ output_text = ii.read()
+ else:
+ output_text = ""
+ output_text += json.dumps(infile) + " -> " + json.dumps(outfile) + "\n"
+ with open(outfile, "w") as oo:
+ oo.write(output_text)
+
+
+
+
+
+
+
+
+class Test_ruffus(unittest.TestCase):
+ def setUp(self):
+ for f in ["a.1", "a.2","a.3","a.4"]:
+ if os.path.exists(f):
+ os.unlink(f)
+
+ def tearDown(self):
+ for f in ["a.1", "a.2","a.3","a.4"]:
+ if os.path.exists(f):
+ os.unlink(f)
+ else:
+ raise Exception("%s is missing" % f)
+
+
+ def test_ruffus (self):
+ pipeline_run(verbose = 0, runtime_data = {"a": runtime_files})
+
+
+ def test_newstyle_ruffus (self):
+
+
+ test_pipeline = Pipeline("test")
+ test_pipeline.originate(task_func = task1,
+ output = ['a.1'] + runtime_files)
+ test_pipeline.transform(task2, task1, suffix(".1"), ".2")
+ test_pipeline.transform(task_func = task3,
+ input = task2,
+ filter = suffix(".2"),
+ output = ".3")
+ test_pipeline.transform(task_func = task4,
+ input = runtime_parameter("a"),
+ filter = suffix(".3"),
+ output = ".4").follows(task3)
+ test_pipeline.run(verbose = 0, runtime_data = {"a": runtime_files})
+
+
+
+if __name__ == '__main__':
+ unittest.main()
+
diff --git a/ruffus/test/test_softlink_uptodate.py b/ruffus/test/test_softlink_uptodate.py
index dc43ef3..5071966 100755
--- a/ruffus/test/test_softlink_uptodate.py
+++ b/ruffus/test/test_softlink_uptodate.py
@@ -1,57 +1,36 @@
#!/usr/bin/env python
from __future__ import print_function
"""
-
test_softlink_uptodate.py
-
- use :
- --debug to test automatically
- -j N / --jobs N to specify multitasking
- -v to see the jobs in action
- -n / --just_print to see what jobs would run
-
"""
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# options
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-import sys, os
-
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-
-
-
-from ruffus import *
-import ruffus.dbdict as dbdict
-
-parser = cmdline.get_argparse( description='Test soft link up to date?', version = "%(prog)s v.2.23")
-options = parser.parse_args()
-
-# optional logger which can be passed to ruffus tasks
-logger, logger_mutex = cmdline.setup_logging (__name__, options.log_file, options.verbose)
+import os
+import sys
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
-# Tasks
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+try:
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
-import multiprocessing.managers
-# list of executed tasks
-manager = multiprocessing.managers.SyncManager()
-manager.start()
-executed_tasks_proxy = manager.dict()
-mutex_proxy = manager.Lock()
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
# Tasks
@@ -62,40 +41,33 @@ mutex_proxy = manager.Lock()
#
# First task
#
- at originate(["a.1", "b.1"], executed_tasks_proxy, mutex_proxy)
-def start_task(output_file_name, executed_tasks_proxy, mutex_proxy):
+ at originate(["a.1", "b.1"])
+def start_task(output_file_name):
with open(output_file_name, "w") as f:
pass
- with mutex_proxy:
- executed_tasks_proxy["start_task"] = 1
#
# Forwards file names, is always as up to date as its input files...
#
- at transform(start_task, suffix(".1"), ".1", executed_tasks_proxy, mutex_proxy)
-def same_file_name_task(input_file_name, output_file_name, executed_tasks_proxy, mutex_proxy):
- with mutex_proxy:
- executed_tasks_proxy["same_file_name_task"] = executed_tasks_proxy.get("same_file_name_task", 0) + 1
+ at transform(start_task, suffix(".1"), ".1")
+def same_file_name_task(input_file_name, output_file_name):
+ pass
#
# Links file names, is always as up to date if links are not missing
#
- at transform(start_task, suffix(".1"), ".linked.1", executed_tasks_proxy, mutex_proxy)
-def linked_file_name_task(input_file_name, output_file_name, executed_tasks_proxy, mutex_proxy):
+ at transform(start_task, suffix(".1"), ".linked.1")
+def linked_file_name_task(input_file_name, output_file_name):
os.symlink(input_file_name, output_file_name)
- with mutex_proxy:
- executed_tasks_proxy["linked_file_name_task"] = executed_tasks_proxy.get("linked_file_name_task", 0) + 1
#
# Final task linking everything
#
- at transform([linked_file_name_task, same_file_name_task], suffix(".1"), ".3", executed_tasks_proxy, mutex_proxy)
-def final_task (input_file_name, output_file_name, executed_tasks_proxy, mutex_proxy):
+ at transform([linked_file_name_task, same_file_name_task], suffix(".1"), ".3")
+def final_task (input_file_name, output_file_name):
with open(output_file_name, "w") as f:
pass
- with mutex_proxy:
- executed_tasks_proxy["final_task"] = executed_tasks_proxy.get("final_task", 0) + 1
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -104,53 +76,42 @@ def final_task (input_file_name, output_file_name, executed_tasks_proxy, mutex_p
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-#
-# Run task 1 only
-#
-logger.debug("Run start_task only")
-options.target_tasks = ["start_task"]
-cmdline.run (options, logger = logger, log_exceptions = True)
-logger.debug("")
+import unittest, shutil
+try:
+ from StringIO import StringIO
+except:
+ from io import StringIO
-#
-# Run task 3 only
-#
-logger.debug("Run final_task: linked_file_name_task should run as well")
-options.target_tasks = []
-cmdline.run (options, logger = logger, log_exceptions = True)
-logger.debug("")
+class Test_ruffus(unittest.TestCase):
+ def setUp(self):
+ for f in ["a.1", "b.1", "a.linked.1", "b.linked.1", "a.3", "b.3", "a.linked.3", "b.linked.3"]:
+ try:
+ os.unlink(f)
+ except:
+ pass
+ def tearDown(self):
+ for f in ["a.1", "b.1", "a.linked.1", "b.linked.1", "a.3", "b.3", "a.linked.3", "b.linked.3"]:
+ if os.path.lexists(f):
+ os.unlink(f)
+ else:
+ raise Exception("Expected %s missing" % f)
-#
-# Run task 3 again:
-#
-# All jobs should be up to date
-#
-logger.debug("Run final_task again: All jobs should be up to date")
-cmdline.run (options, logger = logger, log_exceptions = True)
-logger.debug("")
+ def test_ruffus (self):
+ pipeline_run(log_exceptions = True, verbose = 0)
+ def test_newstyle_ruffus (self):
+ test_pipeline = Pipeline("test")
+ test_pipeline.originate(start_task, ["a.1", "b.1"])
+ test_pipeline.transform(same_file_name_task, start_task, suffix(".1"), ".1")
+ test_pipeline.transform(linked_file_name_task, start_task, suffix(".1"), ".linked.1")
+ test_pipeline.transform(final_task, [linked_file_name_task, same_file_name_task], suffix(".1"), ".3")
+ test_pipeline.run(log_exceptions = True, verbose = 0)
-#
-# cleanup
-#
-for f in ["a.1", "b.1", "a.linked.1", "b.linked.1", "a.3", "b.3", "a.linked.3", "b.linked.3"]:
- if os.path.lexists(f):
- os.unlink(f)
+if __name__ == '__main__':
+ unittest.main()
-#
-# Make sure right number of jobs / tasks ran
-#
-for task_name, jobs_count in ({'start_task': 1, 'final_task': 4, 'linked_file_name_task': 2}).items():
- if task_name not in executed_tasks_proxy:
- raise Exception("Error: %s did not run!!" % task_name)
- if executed_tasks_proxy[task_name] != jobs_count:
- raise Exception("Error: %s did not have %d jobs!!" % (task_name, jobs_count))
-if "same_file_name_task" in executed_tasks_proxy:
- raise Exception("Error: %s should not have run!!" % "same_file_name_task")
-
-print("Succeeded")
diff --git a/ruffus/test/test_split_and_combine.py b/ruffus/test/test_split_and_combine.py
index 7396ea6..56eec56 100755
--- a/ruffus/test/test_split_and_combine.py
+++ b/ruffus/test/test_split_and_combine.py
@@ -2,113 +2,35 @@
from __future__ import print_function
"""
- branching.py
+ test_split_and_combine.py
test branching dependencies
- use :
- --debug to test automatically
- --start_again the first time you run the file
- --jobs_per_task N to simulate tasks with N numbers of files per task
-
- -j N / --jobs N to speify multitasking
- -v to see the jobs in action
- -n / --just_print to see what jobs would run
-
"""
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+import sys
+tempdir = "temp_filesre_split_and_combine/"
+verbose_output = sys.stderr
-# options
+import os
+import sys
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
-from optparse import OptionParser
-import sys, os
-import os.path
-try:
- import StringIO as io
-except:
- import io as io
-import re,time
-
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-if __name__ == '__main__':
- module_name = os.path.split(sys.argv[0])[1]
- module_name = os.path.splitext(module_name)[0];
-else:
- module_name = __name__
-
-
-
-import ruffus
-parser = OptionParser(version="%%prog v1.0, ruffus v%s" % ruffus.ruffus_version.__version)
-parser.add_option("-D", "--debug", dest="debug",
- action="store_true", default=False,
- help="Make sure output is correct and clean up.")
-parser.add_option("-s", "--start_again", dest="start_again",
- action="store_true", default=False,
- help="Make a new 'original.fa' file to simulate having to restart "
- "pipeline from scratch.")
-parser.add_option("--jobs_per_task", dest="jobs_per_task",
- default=50,
- metavar="N",
- type="int",
- help="Simulates tasks with N numbers of files per task.")
-
-
-parser.add_option("-t", "--target_tasks", dest="target_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Target task(s) of pipeline.")
-parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Pipeline task(s) which will be included even if they are up to date.")
-parser.add_option("-j", "--jobs", dest="jobs",
- default=1,
- metavar="jobs",
- type="int",
- help="Specifies the number of jobs (commands) to run simultaneously.")
-parser.add_option("-v", "--verbose", dest = "verbose",
- action="count", default=0,
- help="Print more verbose messages for each additional verbose level.")
-parser.add_option("-d", "--dependency", dest="dependency_file",
- #default="simple.svg",
- metavar="FILE",
- type="string",
- help="Print a dependency graph of the pipeline that would be executed "
- "to FILE, but do not execute it.")
-parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
- metavar="FORMAT",
- type="string",
- default = 'svg',
- help="format of dependency graph file. Can be 'ps' (PostScript), "+
- "'svg' 'svgz' (Structured Vector Graphics), " +
- "'png' 'gif' (bitmap graphics) etc ")
-parser.add_option("-n", "--just_print", dest="just_print",
- action="store_true", default=False,
- help="Print a description of the jobs that would be executed, "
- "but do not execute them.")
-parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
- action="store_true", default=False,
- help="Do not print out legend and key for dependency graph.")
-parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
- action="store_true", default=False,
- help="Draw horizontal dependency graph.")
-
-parameters = [
- ]
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+for attr in "posttask", "split", "merge", "transform", "pipeline_printout", "pipeline_run", "Pipeline", "suffix":
+ globals()[attr] = getattr (ruffus, attr)
@@ -121,49 +43,10 @@ parameters = [
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-import re
-import operator
-import sys,os
-from collections import defaultdict
-import random
-
-sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-
-# use simplejson in place of json for python < 2.6
-try:
- import json
-except ImportError:
- import simplejson
- json = simplejson
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# Main logic
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-
-
-
-
-# get help string
-f =io.StringIO()
-parser.print_help(f)
-helpstr = f.getvalue()
-(options, remaining_args) = parser.parse_args()
-
-
-tempdir = "temp_filesre_split_and_combine/"
-
+import unittest
+import shutil
-if options.verbose:
- verbose_output = sys.stderr
-else:
- verbose_output =open("/dev/null", "w")
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
# Tasks
@@ -173,7 +56,7 @@ else:
#
# split_fasta_file
#
- at posttask(lambda: verbose_output.write("Split into %d files\n" % options.jobs_per_task))
+ at posttask(lambda: verbose_output.write(" Split into %d files\n" % 10))
@split(tempdir + "original.fa", [tempdir + "files.split.success", tempdir + "files.split.*.fa"])
def split_fasta_file (input_file, outputs):
@@ -188,20 +71,23 @@ def split_fasta_file (input_file, outputs):
#
# create as many files as we are simulating in jobs_per_task
#
- for i in range(options.jobs_per_task):
- open(tempdir + "files.split.%03d.fa" % i, "w")
+ for i in range(10):
+ with open(tempdir + "files.split.%03d.fa" % i, "w") as oo:
+ pass
- open(success_flag, "w")
+ with open(success_flag, "w") as oo:
+ pass
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
#
# align_sequences
#
- at posttask(lambda: verbose_output.write("Sequences aligned\n"))
+ at posttask(lambda: verbose_output.write(" Sequences aligned\n"))
@transform(split_fasta_file, suffix(".fa"), ".aln") # fa -> aln
def align_sequences (input_file, output_filename):
- open(output_filename, "w").write("%s\n" % output_filename)
+ with open(output_filename, "w") as oo:
+ oo.write("%s\n" % output_filename)
@@ -209,15 +95,17 @@ def align_sequences (input_file, output_filename):
#
# percentage_identity
#
- at posttask(lambda: verbose_output.write("%Identity calculated\n"))
+ at posttask(lambda: verbose_output.write(" %Identity calculated\n"))
@transform(align_sequences, # find all results from align_sequences
suffix(".aln"), # replace suffix with:
[r".pcid", # .pcid suffix for the result
r".pcid_success"]) # .pcid_success to indicate job completed
def percentage_identity (input_file, output_files):
(output_filename, success_flag_filename) = output_files
- open(output_filename, "w").write("%s\n" % output_filename)
- open(success_flag_filename, "w")
+ with open(output_filename, "w") as oo:
+ oo.write("%s\n" % output_filename)
+ with open(success_flag_filename, "w") as oo:
+ pass
@@ -225,7 +113,7 @@ def percentage_identity (input_file, output_files):
#
# combine_results
#
- at posttask(lambda: verbose_output.write("Results recombined\n"))
+ at posttask(lambda: verbose_output.write(" Results recombined\n"))
@merge(percentage_identity, [tempdir + "all.combine_results",
tempdir + "all.combine_results_success"])
def combine_results (input_files, output_files):
@@ -233,46 +121,79 @@ def combine_results (input_files, output_files):
Combine all
"""
(output_filename, success_flag_filename) = output_files
- out = open(output_filename, "w")
- for inp, flag in input_files:
- out.write(open(inp).read())
- open(success_flag_filename, "w")
+ with open(output_filename, "w") as out:
+ for inp, flag in input_files:
+ with open(inp) as ii:
+ out.write(ii.read())
+ with open(success_flag_filename, "w") as oo:
+ pass
-def start_pipeline_afresh ():
- """
- Recreate directory and starting file
- """
- print("Start again", file=verbose_output)
- import os
- os.system("rm -rf %s" % tempdir)
- os.makedirs(tempdir)
- open(tempdir + "original.fa", "w").close()
+
+
+
+class Test_ruffus(unittest.TestCase):
+ def setUp(self):
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+ os.makedirs(tempdir)
+ open(tempdir + "original.fa", "w").close()
+
+ def tearDown(self):
+ try:
+ shutil.rmtree(tempdir)
+ pass
+ except:
+ pass
+
+ def test_ruffus (self):
+ pipeline_run(multiprocess = 50, verbose = 0)
+ if not os.path.exists(tempdir + "all.combine_results"):
+ raise Exception("Missing %s" % (tempdir + "all.combine_results"))
+
+ def test_newstyle_ruffus (self):
+
+ test_pipeline = Pipeline("test")
+
+
+ test_pipeline.split(task_func = split_fasta_file,
+ input = tempdir + "original.fa",
+ output = [tempdir + "files.split.success",
+ tempdir + "files.split.*.fa"])\
+ .posttask(lambda: verbose_output.write(" Split into %d files\n" % 10))
+
+
+ test_pipeline.transform(task_func = align_sequences,
+ input = split_fasta_file,
+ filter = suffix(".fa"),
+ output = ".aln" # fa -> aln
+ )\
+ .posttask(lambda: verbose_output.write(" Sequences aligned\n"))
+
+ test_pipeline.transform(task_func = percentage_identity,
+ input = align_sequences, # find all results from align_sequences
+ filter = suffix(".aln"), # replace suffix with:
+ output = [r".pcid", # .pcid suffix for the result
+ r".pcid_success"] # .pcid_success to indicate job completed
+ )\
+ .posttask(lambda: verbose_output.write(" %Identity calculated\n"))
+
+
+ test_pipeline.merge(task_func = combine_results,
+ input = percentage_identity,
+ output = [tempdir + "all.combine_results",
+ tempdir + "all.combine_results_success"])\
+ .posttask(lambda: verbose_output.write(" Results recombined\n"))
+
+ test_pipeline.run(multiprocess = 50, verbose = 0)
+ if not os.path.exists(tempdir + "all.combine_results"):
+ raise Exception("Missing %s" % (tempdir + "all.combine_results"))
+
+
if __name__ == '__main__':
- if options.start_again:
- start_pipeline_afresh()
- if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
- verbose = options.verbose)
-
- elif options.dependency_file:
- pipeline_printout_graph ( open(options.dependency_file, "w"),
- options.dependency_graph_format,
- options.target_tasks,
- options.forced_tasks,
- draw_vertically = not options.draw_horizontally,
- no_key_legend = options.no_key_legend_in_graph)
- elif options.debug:
- start_pipeline_afresh()
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- logger = stderr_logger if options.verbose else black_hole_logger,
- verbose = options.verbose)
- os.system("rm -rf %s" % tempdir)
- print("OK")
- else:
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- logger = stderr_logger if options.verbose else black_hole_logger,
- verbose = options.verbose)
+ unittest.main()
diff --git a/ruffus/test/test_split_regex_and_collate.py b/ruffus/test/test_split_regex_and_collate.py
index 737371e..3504e89 100755
--- a/ruffus/test/test_split_regex_and_collate.py
+++ b/ruffus/test/test_split_regex_and_collate.py
@@ -2,10 +2,42 @@
from __future__ import print_function
"""
- branching.py
+ test_split_regex_and_collate.py
"""
+JOBS_PER_TASK = 5
+
+import os
+import sys
+import re
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+
+try:
+ attrlist = ruffus.combinatorics.__all__
+
+except AttributeError:
+ attrlist = dir (ruffus.combinatorics)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus.combinatorics, attr)
+for attr in "collate", "pipeline_run", "pipeline_printout", "suffix", "transform", "split", "merge", "dbdict", "follows", "mkdir", "originate", "posttask", "subdivide", "regex", "Pipeline":
+ globals()[attr] = getattr (ruffus, attr)
+RethrownJobError = ruffus.ruffus_exceptions.RethrownJobError
+RUFFUS_HISTORY_FILE = ruffus.ruffus_utility.RUFFUS_HISTORY_FILE
+CHECKSUM_FILE_TIMESTAMPS = ruffus.ruffus_utility.CHECKSUM_FILE_TIMESTAMPS
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -14,32 +46,14 @@ from __future__ import print_function
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-from optparse import OptionParser
-import sys, os
-import os.path
try:
from StringIO import StringIO
except:
from io import StringIO
-import re,time
-import operator
-from collections import defaultdict
-import random
-import shutil
-
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0, os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-from ruffus import (pipeline_run, pipeline_printout, suffix, transform, split,
- merge, dbdict, follows)
-from ruffus.combinatorics import *
-from ruffus.ruffus_exceptions import RethrownJobError
-from ruffus.ruffus_utility import (RUFFUS_HISTORY_FILE,
- CHECKSUM_FILE_TIMESTAMPS)
+import shutil
import unittest
-JOBS_PER_TASK = 5
@@ -173,7 +187,7 @@ def combine_results (input_files, output_files):
-class Test_split_regex_and_collate(unittest.TestCase):
+class Test_ruffus(unittest.TestCase):
def setUp(self):
import os
try:
@@ -196,11 +210,67 @@ class Test_split_regex_and_collate(unittest.TestCase):
s = StringIO()
pipeline_printout(s, [combine_results], verbose=5, wrap_width = 10000)
- self.assertTrue('Job needs update: Missing files\n' in s.getvalue())
+ self.assertTrue(re.search('Job needs update:.*Missing files.*', s.getvalue(), re.DOTALL) is not None)
#print s.getvalue()
pipeline_run([combine_results], verbose=0)
+
+ def test_newstyle_collate (self):
+ """
+ As above but create pipeline on the fly using object orientated syntax rather than decorators
+ """
+
+ #
+ # Create pipeline on the fly, joining up tasks
+ #
+ test_pipeline = Pipeline("test")
+
+ test_pipeline.originate(task_func = generate_initial_files,
+ output = original_files)\
+ .mkdir(tempdir, tempdir+"/test")
+
+
+ test_pipeline.subdivide( task_func = split_fasta_file,
+ input = generate_initial_files,
+ filter = regex(r".*\/original_(\d+).fa"), # match original files
+ output = [tempdir + r"/files.split.\1.success", # flag file for each original file
+ tempdir + r"/files.split.\1.*.fa"], # glob pattern
+ extras = [r"\1"])\
+ .posttask(lambda: sys.stderr.write("\tSplit into %d files each\n" % JOBS_PER_TASK))
+
+
+ test_pipeline.transform(task_func = align_sequences,
+ input = split_fasta_file,
+ filter = suffix(".fa"),
+ output = ".aln") \
+ .posttask(lambda: sys.stderr.write("\tSequences aligned\n"))
+
+ test_pipeline.transform(task_func = percentage_identity,
+ input = align_sequences, # find all results from align_sequences
+ filter = suffix(".aln"), # replace suffix with:
+ output = [r".pcid", # .pcid suffix for the result
+ r".pcid_success"] # .pcid_success to indicate job completed
+ )\
+ .posttask(lambda: sys.stderr.write("\t%Identity calculated\n"))
+
+
+ test_pipeline.collate(task_func = combine_results,
+ input = percentage_identity,
+ filter = regex(r".*files.split\.(\d+)\.\d+.pcid"),
+ output = [tempdir + r"/\1.all.combine_results",
+ tempdir + r"/\1.all.combine_results_success"])\
+ .posttask(lambda: sys.stderr.write("\tResults recombined\n"))
+
+ #
+ # Cleanup, printout and run
+ #
+ self.cleanup_tmpdir()
+ s = StringIO()
+ test_pipeline.printout(s, [combine_results], verbose=5, wrap_width = 10000)
+ self.assertTrue(re.search('Job needs update:.*Missing files.*', s.getvalue(), re.DOTALL) is not None)
+ test_pipeline.run(verbose=0)
+
#___________________________________________________________________________
#
# cleanup
@@ -213,6 +283,5 @@ class Test_split_regex_and_collate(unittest.TestCase):
# see: http://docs.python.org/library/multiprocessing.html#multiprocessing-programming
#
if __name__ == '__main__':
-# pipeline_printout(sys.stdout, [combine_results], verbose = 5)
unittest.main()
diff --git a/ruffus/test/test_split_subdivide_checkpointing.py b/ruffus/test/test_split_subdivide_checkpointing.py
new file mode 100755
index 0000000..95c7e4d
--- /dev/null
+++ b/ruffus/test/test_split_subdivide_checkpointing.py
@@ -0,0 +1,239 @@
+#!/usr/bin/env python
+from __future__ import print_function
+"""
+
+ test_split_subdivide_checkpointing.py
+
+
+"""
+tempdir = "testing_dir/"
+
+
+import os
+import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+for attr in "pipeline_run", "pipeline_printout", "originate", "split", "transform", "subdivide", "formatter", "Pipeline":
+ globals()[attr] = getattr (ruffus, attr)
+
+
+
+
+
+
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# imports
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+import unittest
+import shutil
+
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# Each time the pipeline is FORCED to rerun,
+# More files are created for each task
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+
+
+
+ at originate([tempdir + 'start'])
+def make_start(outfile):
+ """
+ -> start
+ """
+ open(outfile, 'w').close()
+
+
+ at split(make_start, tempdir + '*.split')
+def split_start(infiles, outfiles):
+ """
+ -> XXX.split
+ where XXX = 0 .. N,
+ N = previous N + 1
+ """
+
+ # split always runs exactly one job (unlike @subdivide)
+ # So it implicitly combines all its inputs before running and generating multiple output
+ # @originate generates multiple output so the input for @split is a list...
+ infile = infiles[0]
+
+ # clean up previous
+ for f in outfiles:
+ os.unlink(f)
+
+
+ #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ #
+ # Create more files than the previous invocation
+ #
+ #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ n_to_produce = len(outfiles) + 1
+ for i in range(n_to_produce):
+ f = '{}{}.split'.format(tempdir, i)
+ open(f, 'a').close()
+
+
+
+ at subdivide(split_start, formatter(), tempdir + '{basename[0]}_*.subdivided', tempdir + '{basename[0]}')
+def subdivide_start(infile, outfiles, infile_basename):
+ # cleanup existing
+ for f in outfiles:
+ os.unlink(f)
+
+ #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ #
+ # Create more files than the previous invocation
+ #
+ #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ n_to_produce = len(outfiles) + 1
+ for i in range( n_to_produce):
+ open('{}_{}.subdivided'.format(infile_basename, i), 'a').close()
+
+
+
+
+class Test_ruffus(unittest.TestCase):
+
+ def tearDown(self):
+ # only tear down if not throw exception so we can debug?
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+
+ def setUp(self):
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+ os.makedirs(tempdir)
+
+ def check_file_exists_or_not_as_expected(self, expected_files, not_expected_files):
+ """
+ Check if files exist / not exist
+ """
+ for ee in expected_files:
+ if not os.path.exists(tempdir + ee):
+ raise Exception("Expected file %s" % (tempdir + ee))
+ for ne in not_expected_files:
+ if os.path.exists(tempdir + ne):
+ raise Exception("Unexpected file %s" % (tempdir + ne ))
+
+ def test_newstyle_ruffus (self):
+
+ test_pipeline = Pipeline("test")
+ test_pipeline.originate(task_func = make_start, output = [tempdir + 'start'])
+ test_pipeline.split(task_func = split_start, input = make_start, output = tempdir + '*.split')
+ test_pipeline.subdivide(task_func = subdivide_start, input = split_start, filter = formatter(), output = tempdir + '{basename[0]}_*.subdivided', extras = [tempdir + '{basename[0]}'])
+
+ expected_files_after_1_runs = ["start", "0.split", "0_0.subdivided"]
+ expected_files_after_2_runs = ["1.split", "0_1.subdivided", "1_0.subdivided"]
+ expected_files_after_3_runs = ["2.split", "0_2.subdivided", "1_1.subdivided", "2_0.subdivided"]
+ expected_files_after_4_runs = ["3.split", "0_3.subdivided", "1_2.subdivided", "2_1.subdivided", "3_0.subdivided"]
+
+ print(" Run pipeline normally...")
+ test_pipeline.run(multiprocess = 10, verbose=0)
+ self.check_file_exists_or_not_as_expected(expected_files_after_1_runs,
+ expected_files_after_2_runs)
+
+ print(" Check that running again does nothing. (All up to date).")
+ test_pipeline.run(multiprocess = 10, verbose=0)
+ self.check_file_exists_or_not_as_expected(expected_files_after_1_runs,
+ expected_files_after_2_runs)
+
+ print(" Running again with forced tasks to generate more files...")
+ test_pipeline.run(forcedtorun_tasks = ["test::make_start"], multiprocess = 10, verbose=0)
+ self.check_file_exists_or_not_as_expected(expected_files_after_1_runs
+ + expected_files_after_2_runs,
+ expected_files_after_3_runs)
+
+ print(" Check that running again does nothing. (All up to date).")
+ test_pipeline.run(multiprocess = 10, verbose=0)
+ self.check_file_exists_or_not_as_expected(expected_files_after_1_runs
+ + expected_files_after_2_runs,
+ expected_files_after_3_runs)
+
+
+ print(" Running again with forced tasks to generate even more files...")
+ test_pipeline.run(forcedtorun_tasks = make_start, multiprocess = 10, verbose=0)
+ self.check_file_exists_or_not_as_expected(expected_files_after_1_runs
+ + expected_files_after_2_runs
+ + expected_files_after_3_runs,
+ expected_files_after_4_runs)
+ print(" Check that running again does nothing. (All up to date).")
+ test_pipeline.run(multiprocess = 10, verbose=0)
+ self.check_file_exists_or_not_as_expected(expected_files_after_1_runs
+ + expected_files_after_2_runs
+ + expected_files_after_3_runs,
+ expected_files_after_4_runs)
+
+
+ def test_ruffus (self):
+
+ expected_files_after_1_runs = ["start", "0.split", "0_0.subdivided"]
+ expected_files_after_2_runs = ["1.split", "0_1.subdivided", "1_0.subdivided"]
+ expected_files_after_3_runs = ["2.split", "0_2.subdivided", "1_1.subdivided", "2_0.subdivided"]
+ expected_files_after_4_runs = ["3.split", "0_3.subdivided", "1_2.subdivided", "2_1.subdivided", "3_0.subdivided"]
+
+ print(" Run pipeline normally...")
+ pipeline_run(multiprocess = 10, verbose=0)
+ self.check_file_exists_or_not_as_expected(expected_files_after_1_runs,
+ expected_files_after_2_runs)
+
+ print(" Check that running again does nothing. (All up to date).")
+ pipeline_run(multiprocess = 10, verbose=0)
+ self.check_file_exists_or_not_as_expected(expected_files_after_1_runs,
+ expected_files_after_2_runs)
+
+ print(" Running again with forced tasks to generate more files...")
+ pipeline_run(forcedtorun_tasks = [make_start], multiprocess = 10, verbose=0)
+ self.check_file_exists_or_not_as_expected(expected_files_after_1_runs
+ + expected_files_after_2_runs,
+ expected_files_after_3_runs)
+
+ print(" Check that running again does nothing. (All up to date).")
+ pipeline_run(multiprocess = 10, verbose=0)
+ self.check_file_exists_or_not_as_expected(expected_files_after_1_runs
+ + expected_files_after_2_runs,
+ expected_files_after_3_runs)
+
+
+ print(" Running again with forced tasks to generate even more files...")
+ pipeline_run(forcedtorun_tasks = [make_start], multiprocess = 10, verbose=0)
+ self.check_file_exists_or_not_as_expected(expected_files_after_1_runs
+ + expected_files_after_2_runs
+ + expected_files_after_3_runs,
+ expected_files_after_4_runs)
+ print(" Check that running again does nothing. (All up to date).")
+ pipeline_run(multiprocess = 10, verbose=0)
+ self.check_file_exists_or_not_as_expected(expected_files_after_1_runs
+ + expected_files_after_2_runs
+ + expected_files_after_3_runs,
+ expected_files_after_4_runs)
+
+
+if __name__ == '__main__':
+ unittest.main()
+
+
diff --git a/ruffus/test/test_subpipeline.py b/ruffus/test/test_subpipeline.py
new file mode 100755
index 0000000..01a86d7
--- /dev/null
+++ b/ruffus/test/test_subpipeline.py
@@ -0,0 +1,201 @@
+#!/usr/bin/env python
+from __future__ import print_function
+"""
+ test_subpipeline.py
+
+ Demonstrates the new Ruffus syntax in version 2.6
+"""
+
+import os
+import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+import ruffus
+from ruffus import add_inputs, suffix, mkdir, regex, Pipeline, output_from, touch_file
+print("\tRuffus Version = ", ruffus.__version__)
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# imports
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+import shutil
+
+
+def touch (outfile):
+ with open(outfile, "w"):
+ pass
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# Tasks
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+tempdir = "tempdir/"
+def task_originate(o):
+ """
+ Makes new files
+ """
+ touch(o)
+
+def task_m_to_1(i, o):
+ """
+ Merges files together
+ """
+ with open(o, "w") as o_file:
+ for f in sorted(i):
+ with open(f) as ii:
+ o_file.write(f +"=" + ii.read() + "; ")
+
+def task_1_to_1(i, o):
+ """
+ 1 to 1 for transform
+ """
+ with open(o, "w") as o_file:
+ with open(i) as ii:
+ o_file.write(i +"+" + ii.read())
+
+DEBUG_do_not_define_tail_task = False
+DEBUG_do_not_define_head_task = False
+
+import unittest
+
+#
+# Returns a fully formed sub pipeline useable as a building block
+#
+def make_pipeline1(pipeline_name, # Pipelines need to have a unique name
+ starting_file_names):
+ test_pipeline = Pipeline(pipeline_name)
+
+ # We can change the starting files later using
+ # set_input() for transform etc.
+ # or set_output() for originate
+ # But it can be more convenient to just pass this to the function making the pipeline
+ #
+ test_pipeline.originate(task_originate, starting_file_names)\
+ .follows(mkdir(tempdir), mkdir(tempdir + "testdir", tempdir + "testdir2"))\
+ .posttask(touch_file(tempdir + "testdir/whatever.txt"))
+ test_pipeline.transform(task_func = task_m_to_1,
+ name = "add_input",
+ # Lookup Task from function name task_originate()
+ # So long as this is unique in the pipeline
+ input = task_originate,
+ filter = regex(r"(.*)"),
+ add_inputs = add_inputs(tempdir + "testdir/whatever.txt"),
+ output = r"\1.22")
+ test_pipeline.transform(task_func = task_1_to_1,
+ name = "22_to_33",
+ # Lookup Task from Task name
+ # Function name is not unique in the pipeline
+ input = output_from("add_input"),
+ filter = suffix(".22"),
+ output = ".33")
+ tail_task = test_pipeline.transform(task_func = task_1_to_1,
+ name = "33_to_44",
+ # Ask Pipeline to lookup Task from Task name
+ input = test_pipeline["22_to_33"],
+ filter = suffix(".33"),
+ output = ".44")
+
+ # Set the tail task so that users of my sub pipeline can use it as a dependency
+ # without knowing the details of task names
+ #
+ # Use Task() object directly without having to lookup
+ test_pipeline.set_tail_tasks([tail_task])
+
+ # If we try to connect a Pipeline without tail tasks defined, we have to
+ # specify the exact task within the Pipeline.
+ # Otherwise Ruffus will not know which task we mean and throw an exception
+ if DEBUG_do_not_define_tail_task:
+ test_pipeline.set_tail_tasks([])
+
+ # Set the head task so that users of my sub pipeline send input into it
+ # without knowing the details of task names
+ test_pipeline.set_head_tasks([test_pipeline[task_originate]])
+
+ return test_pipeline
+
+#
+# Returns a fully formed sub pipeline useable as a building block
+#
+def make_pipeline2( pipeline_name = "pipeline2"):
+ test_pipeline2 = Pipeline(pipeline_name)
+ test_pipeline2.transform(task_func = task_1_to_1,
+ # task name
+ name = "44_to_55",
+ # placeholder: will be replaced later with set_input()
+ input = None,
+ filter = suffix(".44"),
+ output = ".55")
+ test_pipeline2.merge( task_func = task_m_to_1,
+ input = test_pipeline2["44_to_55"],
+ output = tempdir + "final.output",)
+
+ # Set head and tail
+ test_pipeline2.set_tail_tasks([test_pipeline2[task_m_to_1]])
+ if not DEBUG_do_not_define_head_task:
+ test_pipeline2.set_head_tasks([test_pipeline2["44_to_55"]])
+
+ return test_pipeline2
+
+
+def run_pipeline():
+
+ # First two pipelines are created as separate instances by the make_pipeline1 function
+ pipeline1a = make_pipeline1(pipeline_name = "pipeline1a", starting_file_names = [tempdir + ss for ss in ("a.1", "b.1")])
+ pipeline1b = make_pipeline1(pipeline_name = "pipeline1b", starting_file_names = [tempdir + ss for ss in ("c.1", "d.1")])
+
+ # The Third pipeline is a clone of pipeline1b
+ pipeline1c = pipeline1b.clone(new_name = "pipeline1c")
+
+ # Set the "originate" files for pipeline1c to ("e.1" and "f.1")
+ # Otherwise they would use the original ("c.1", "d.1")
+ pipeline1c.set_output(output = [])
+ pipeline1c.set_output(output = [tempdir + ss for ss in ("e.1", "f.1")])
+
+ # Join all pipeline1a-c to pipeline2
+ pipeline2 = make_pipeline2()
+ pipeline2.set_input(input = [pipeline1a, pipeline1b, pipeline1c])
+
+
+ #pipeline2.printout_graph("test.svg", "svg", [task_m_to_1])
+ #pipeline2.printout(verbose = 0)
+ pipeline2.run(multiprocess = 10, verbose = 0)
+
+
+class Test_task(unittest.TestCase):
+
+ def tearDown (self):
+ """
+ """
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
+
+
+ def test_subpipelines (self):
+
+ run_pipeline()
+
+ # Check that the output reflecting the pipeline topology is correct.
+ correct_output = 'tempdir/a.1.55=tempdir/a.1.44+tempdir/a.1.33+tempdir/a.1.22+tempdir/a.1=; tempdir/testdir/whatever.txt=; ; ' \
+ 'tempdir/b.1.55=tempdir/b.1.44+tempdir/b.1.33+tempdir/b.1.22+tempdir/b.1=; tempdir/testdir/whatever.txt=; ; ' \
+ 'tempdir/c.1.55=tempdir/c.1.44+tempdir/c.1.33+tempdir/c.1.22+tempdir/c.1=; tempdir/testdir/whatever.txt=; ; ' \
+ 'tempdir/d.1.55=tempdir/d.1.44+tempdir/d.1.33+tempdir/d.1.22+tempdir/d.1=; tempdir/testdir/whatever.txt=; ; ' \
+ 'tempdir/e.1.55=tempdir/e.1.44+tempdir/e.1.33+tempdir/e.1.22+tempdir/e.1=; tempdir/testdir/whatever.txt=; ; ' \
+ 'tempdir/f.1.55=tempdir/f.1.44+tempdir/f.1.33+tempdir/f.1.22+tempdir/f.1=; tempdir/testdir/whatever.txt=; ; '
+ with open(tempdir + "final.output") as real_output:
+ real_output_str = real_output.read()
+ self.assertEqual(correct_output, real_output_str)
+
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/ruffus/test/test_suffix_output_dir.py b/ruffus/test/test_suffix_output_dir.py
new file mode 100755
index 0000000..d20b449
--- /dev/null
+++ b/ruffus/test/test_suffix_output_dir.py
@@ -0,0 +1,259 @@
+#!/usr/bin/env python
+from __future__ import print_function
+"""
+
+ test_suffix_output_dir.py
+
+"""
+
+import os
+import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+
+try:
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
+
+
+
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# imports
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+import unittest
+import shutil
+import json
+
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# Main logic
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+def helper (infiles, outfiles):
+ if not isinstance(infiles, (tuple, list)):
+ infiles = [infiles]
+ if not isinstance(outfiles, list):
+ outfiles = [outfiles]
+
+ output_text = ""
+ preamble_len = 0
+ for infile in infiles:
+ if infile:
+ with open(infile) as ii:
+ for line in ii:
+ output_text += line
+ preamble_len = max(preamble_len, len(line) - len(line.lstrip()))
+
+ preamble = " " * (preamble_len + 4) if len(output_text) else ""
+
+ for outfile in outfiles:
+ file_output_text = preamble + json.dumps(infiles) + " -> " + json.dumps(outfiles) + "\n"
+ with open(outfile, "w") as oo:
+ oo.write(output_text + file_output_text)
+
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+# Tasks
+
+
+#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+
+#
+# task1
+#
+root_dir = "test_suffix_output_dir"
+data_dir = "test_suffix_output_dir/data"
+work_dir = "test_suffix_output_dir/work"
+ at mkdir(data_dir, work_dir)
+ at originate([os.path.join(data_dir, "%s.1" % aa) for aa in "abcd"])
+def task1(outfile):
+ """
+ First task
+ """
+ # N.B. originate works with an extra parameter
+ helper (None, outfile)
+
+
+
+#
+# task2
+#
+ at mkdir( task1,
+ suffix(".1"),
+ ".dir",
+ output_dir = work_dir)
+ at transform(task1, suffix(".1"), ".1", "extra.tst", 4, r"orig_dir=\1", output_dir = work_dir)
+def task2(infile, outfile, extra_str, extra_num, extra_dir):
+ """
+ Second task
+ """
+ if (extra_str, extra_num) != ("extra.tst", 4) and \
+ extra_dir[:len("orig_dir=" + data_dir)] != "orig_dir=" + data_dir:
+ raise Exception("transform with output_dir has changed extras")
+ helper (infile, outfile)
+
+
+#
+# task3
+#
+ at subdivide(task2, suffix(".1"), r"\1.*.2", [r"\1.a.2", r"\1.b.2"])
+def task3(infile, ignore_outfiles, outfiles):
+ """
+ Third task
+ """
+ helper (infile, outfiles)
+
+
+
+#
+# task4
+#
+ at transform(task3, suffix(".2"), ".3", output_dir = work_dir)
+def task4(infile, outfile):
+ """
+ Fourth task
+ """
+ helper (infile, outfile)
+
+#
+# task4
+#
+ at merge(task4, os.path.join(data_dir, "summary.5"))
+def task5(infiles, outfile):
+ """
+ Fifth task
+ """
+ helper (infiles, outfile)
+
+
+expected_active_text = """[null] -> ["test_suffix_output_dir/data/a.1"]
+ ["test_suffix_output_dir/data/a.1"] -> ["test_suffix_output_dir/work/a.1"]
+ ["test_suffix_output_dir/work/a.1"] -> ["test_suffix_output_dir/work/a.a.2", "test_suffix_output_dir/work/a.b.2"]
+ ["test_suffix_output_dir/work/a.a.2"] -> ["test_suffix_output_dir/work/a.a.3"]
+[null] -> ["test_suffix_output_dir/data/a.1"]
+ ["test_suffix_output_dir/data/a.1"] -> ["test_suffix_output_dir/work/a.1"]
+ ["test_suffix_output_dir/work/a.1"] -> ["test_suffix_output_dir/work/a.a.2", "test_suffix_output_dir/work/a.b.2"]
+ ["test_suffix_output_dir/work/a.b.2"] -> ["test_suffix_output_dir/work/a.b.3"]
+[null] -> ["test_suffix_output_dir/data/b.1"]
+ ["test_suffix_output_dir/data/b.1"] -> ["test_suffix_output_dir/work/b.1"]
+ ["test_suffix_output_dir/work/b.1"] -> ["test_suffix_output_dir/work/b.a.2", "test_suffix_output_dir/work/b.b.2"]
+ ["test_suffix_output_dir/work/b.a.2"] -> ["test_suffix_output_dir/work/b.a.3"]
+[null] -> ["test_suffix_output_dir/data/b.1"]
+ ["test_suffix_output_dir/data/b.1"] -> ["test_suffix_output_dir/work/b.1"]
+ ["test_suffix_output_dir/work/b.1"] -> ["test_suffix_output_dir/work/b.a.2", "test_suffix_output_dir/work/b.b.2"]
+ ["test_suffix_output_dir/work/b.b.2"] -> ["test_suffix_output_dir/work/b.b.3"]
+[null] -> ["test_suffix_output_dir/data/c.1"]
+ ["test_suffix_output_dir/data/c.1"] -> ["test_suffix_output_dir/work/c.1"]
+ ["test_suffix_output_dir/work/c.1"] -> ["test_suffix_output_dir/work/c.a.2", "test_suffix_output_dir/work/c.b.2"]
+ ["test_suffix_output_dir/work/c.a.2"] -> ["test_suffix_output_dir/work/c.a.3"]
+[null] -> ["test_suffix_output_dir/data/c.1"]
+ ["test_suffix_output_dir/data/c.1"] -> ["test_suffix_output_dir/work/c.1"]
+ ["test_suffix_output_dir/work/c.1"] -> ["test_suffix_output_dir/work/c.a.2", "test_suffix_output_dir/work/c.b.2"]
+ ["test_suffix_output_dir/work/c.b.2"] -> ["test_suffix_output_dir/work/c.b.3"]
+[null] -> ["test_suffix_output_dir/data/d.1"]
+ ["test_suffix_output_dir/data/d.1"] -> ["test_suffix_output_dir/work/d.1"]
+ ["test_suffix_output_dir/work/d.1"] -> ["test_suffix_output_dir/work/d.a.2", "test_suffix_output_dir/work/d.b.2"]
+ ["test_suffix_output_dir/work/d.a.2"] -> ["test_suffix_output_dir/work/d.a.3"]
+[null] -> ["test_suffix_output_dir/data/d.1"]
+ ["test_suffix_output_dir/data/d.1"] -> ["test_suffix_output_dir/work/d.1"]
+ ["test_suffix_output_dir/work/d.1"] -> ["test_suffix_output_dir/work/d.a.2", "test_suffix_output_dir/work/d.b.2"]
+ ["test_suffix_output_dir/work/d.b.2"] -> ["test_suffix_output_dir/work/d.b.3"]
+ ["test_suffix_output_dir/work/a.a.3", "test_suffix_output_dir/work/a.b.3", "test_suffix_output_dir/work/b.a.3", "test_suffix_output_dir/work/b.b.3", "test_suffix_output_dir/work/c.a.3", "test_suffix_output_dir/work/c.b.3", "test_suffix_output_dir/work/d.a.3", "test_suffix_output_dir/work/d.b.3"] -> ["test_suffix_output_dir/data/summary.5"]
+"""
+
+
+
+
+
+
+class Test_ruffus(unittest.TestCase):
+ def setUp(self):
+ try:
+ shutil.rmtree(root_dir)
+ except:
+ pass
+ for tempdir in root_dir, work_dir, data_dir:
+ try:
+ os.makedirs(tempdir)
+ except:
+ pass
+
+
+
+ def tearDown(self):
+ try:
+ shutil.rmtree(root_dir)
+ except:
+ sys.stderr.write("Can't remove %s" % root_dir)
+ pass
+
+ def test_ruffus (self):
+ pipeline_run(multiprocess = 50, verbose = 0)
+
+ with open(os.path.join(data_dir, "summary.5")) as ii:
+ active_text = ii.read()
+ if active_text != expected_active_text:
+ raise Exception("Error:\n\tExpected\n%s\nInstead\n%s\n" % (expected_active_text, active_text))
+
+ def test_newstyle_ruffus (self):
+ # alternative syntax
+ test_pipeline = Pipeline("test")
+
+
+ test_pipeline.mkdir(data_dir, work_dir)
+ test_pipeline.originate(task_func = task1,
+ output = [os.path.join(data_dir, "%s.1" % aa) for aa in "abcd"])
+
+ test_pipeline.mkdir(filter = suffix(".1"),
+ output = ".dir",
+ output_dir = work_dir)
+
+ test_pipeline.transform(task_func = task2,
+ input = task1,
+ filter = suffix(".1"),
+ output = ".1",
+ extras = ["extra.tst", 4, r"orig_dir=\1"],
+ output_dir = work_dir)
+
+ test_pipeline.subdivide(task3, task2, suffix(".1"), r"\1.*.2", [r"\1.a.2", r"\1.b.2"])
+ test_pipeline.transform(task4, task3, suffix(".2"), ".3", output_dir = work_dir)
+ test_pipeline.merge(task5, task4, os.path.join(data_dir, "summary.5"))
+ test_pipeline.run(multiprocess = 50, verbose = 0)
+
+ with open(os.path.join(data_dir, "summary.5")) as ii:
+ active_text = ii.read()
+ if active_text != expected_active_text:
+ raise Exception("Error:\n\tExpected\n%s\nInstead\n%s\n" % (expected_active_text, active_text))
+
+
+
+if __name__ == '__main__':
+ unittest.main()
+
diff --git a/ruffus/test/test_task_file_dependencies.py b/ruffus/test/test_task_file_dependencies.py
index 5874399..df90f13 100755
--- a/ruffus/test/test_task_file_dependencies.py
+++ b/ruffus/test/test_task_file_dependencies.py
@@ -4,49 +4,43 @@ from __future__ import print_function
#
# test_task_file_dependencies.py
#
-#
-# Copyright (c) 2009 Leo Goodstadt
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
#################################################################################
"""
test_task_file_dependencies.py
"""
+history_file = ':memory:'
+history_file = False
-# use simplejson in place of json for python < 2.6
-try:
- import json
-except ImportError:
- import simplejson
- json = simplejson
-import unittest, os,sys
-if __name__ != '__main__':
- raise Exception ("This is not a callable module [%s]" % __main__)
+import os
+import sys
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0, os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-from ruffus.ruffus_utility import open_job_history
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+for attr in "parallel", "pipeline_run", "Pipeline", "task":
+ globals()[attr] = getattr (ruffus, attr)
+open_job_history = ruffus.file_name_parameters.open_job_history
+CHECKSUM_HISTORY_TIMESTAMPS = ruffus.ruffus_utility.CHECKSUM_HISTORY_TIMESTAMPS
+
+
+import unittest
+
+
+class dummy_task (object):
+ checksum_level = CHECKSUM_HISTORY_TIMESTAMPS
+ def user_defined_work_func(self):
+ pass
-history_file = ':memory:'
-history_file = None
class Test_needs_update_check_modify_time(unittest.TestCase):
@@ -56,6 +50,7 @@ class Test_needs_update_check_modify_time(unittest.TestCase):
"""
import tempfile,time
self.files = list()
+ job_history = open_job_history(history_file)
for i in range(6):
#test_file =tempfile.NamedTemporaryFile(delete=False, prefix='testing_tmp')
#self.files.append (test_file.name)
@@ -64,7 +59,22 @@ class Test_needs_update_check_modify_time(unittest.TestCase):
fh, temp_file_name = tempfile.mkstemp(suffix='.dot')
self.files.append (temp_file_name)
os.fdopen(fh, "w").close()
- time.sleep(0.1)
+
+ # Save modify time in history file
+ mtime = os.path.getmtime(temp_file_name)
+ epoch_seconds = time.time()
+ # Use epoch seconds unless there is a > 1 second discrepancy between system clock
+ # and file system clock
+ if epoch_seconds > mtime and epoch_seconds - mtime < 1.1:
+ mtime = epoch_seconds
+ else:
+ # file system clock out of sync:
+ # Use file modify times: slow down in case of low counter resolution
+ # (e.g. old versions of NFS and windows)
+ time.sleep(2)
+ chksum = task.JobHistoryChecksum(temp_file_name, mtime, "", dummy_task())
+ job_history[os.path.relpath(temp_file_name)] = chksum
+
def tearDown (self):
"""
@@ -80,29 +90,38 @@ class Test_needs_update_check_modify_time(unittest.TestCase):
#
self.assertTrue(not task.needs_update_check_modify_time (self.files[0:2],
self.files[2:6],
- job_history = open_job_history(history_file))[0])
+ job_history = open_job_history(history_file),
+ task = dummy_task())[0])
self.assertTrue( task.needs_update_check_modify_time (self.files[2:6],
self.files[0:2],
- job_history = open_job_history(history_file))[0])
+ job_history = open_job_history(history_file),
+ task = dummy_task())[0])
#
# singletons and lists of files
#
self.assertTrue(not task.needs_update_check_modify_time (self.files[0],
self.files[2:6],
- job_history = open_job_history(history_file))[0])
+ job_history = open_job_history(history_file),
+ task = dummy_task())[0])
self.assertTrue( task.needs_update_check_modify_time (self.files[2:6],
self.files[0],
- job_history = open_job_history(history_file))[0])
+ job_history = open_job_history(history_file),
+ task = dummy_task())[0])
+
#
# singletons
#
self.assertTrue( task.needs_update_check_modify_time (self.files[3],
self.files[0],
- job_history = open_job_history(history_file))[0])
+ job_history = open_job_history(history_file),
+ task = dummy_task())[0])
+
# self -self = no update
self.assertTrue(not task.needs_update_check_modify_time (self.files[0],
self.files[0],
- job_history = open_job_history(history_file))[0])
+ job_history = open_job_history(history_file),
+ task = dummy_task())[0])
+
#
# missing files means need update
@@ -110,23 +129,31 @@ class Test_needs_update_check_modify_time(unittest.TestCase):
self.assertTrue( task.needs_update_check_modify_time (self.files[0:2] +
["uncreated"],
self.files[3:6],
- job_history = open_job_history(history_file))[0])
+ job_history = open_job_history(history_file),
+ task = dummy_task())[0])
+
self.assertTrue( task.needs_update_check_modify_time (self.files[0:2],
self.files[3:6] +
["uncreated"],
- job_history = open_job_history(history_file))[0])
+ job_history = open_job_history(history_file),
+ task = dummy_task())[0])
+
#
# None means need update
#
self.assertTrue( task.needs_update_check_modify_time (self.files[0:2],
None,
- job_history = open_job_history(history_file))[0])
+ job_history = open_job_history(history_file),
+ task = dummy_task())[0])
+
#
# None input means need update only if do not exist
#
self.assertTrue( not task.needs_update_check_modify_time (None,
self.files[3:6],
- job_history = open_job_history(history_file))[0])
+ job_history = open_job_history(history_file),
+ task = dummy_task())[0])
+
#
@@ -135,15 +162,20 @@ class Test_needs_update_check_modify_time(unittest.TestCase):
self.assertTrue( task.needs_update_check_modify_time (self.files[0:2] +
["uncreated"],
None,
- job_history = open_job_history(history_file))[0])
+ job_history = open_job_history(history_file),
+ task = dummy_task())[0])
+
self.assertTrue( task.needs_update_check_modify_time (None,
self.files[3:6] +
["uncreated"],
- job_history = open_job_history(history_file))[0])
+ job_history = open_job_history(history_file),
+ task = dummy_task())[0])
+
-unittest.main()
+if __name__ == '__main__':
+ unittest.main()
diff --git a/ruffus/test/test_task_misc.py b/ruffus/test/test_task_misc.py
index c607630..2a7f360 100755
--- a/ruffus/test/test_task_misc.py
+++ b/ruffus/test/test_task_misc.py
@@ -4,24 +4,32 @@ from __future__ import print_function
test_task_misc.py
"""
-# use simplejson in place of json for python < 2.6
-try:
- import json
-except ImportError:
- import simplejson
- json = simplejson
-import unittest, os,sys
-if __name__ != '__main__':
- raise Exception ("This is not a callable module [%s]" % __main__)
-
-
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0, os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-
-
-
-
+import os
+import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+
+for attr in "task",:
+ globals()[attr] = getattr (ruffus, attr)
+
+
+
+import unittest
+
+
+
+
class Test_needs_update_check_directory_missing(unittest.TestCase):
def setUp (self):
@@ -36,26 +44,27 @@ class Test_needs_update_check_directory_missing(unittest.TestCase):
fh, self.tempfile = tempfile.mkstemp(suffix='.dot')
os.fdopen(fh, "w").close()
self.directory = tempfile.mkdtemp(prefix='testing_tmp')
-
+
def tearDown (self):
"""
delete files
"""
os.unlink(self.tempfile)
- os.removedirs(self.directory)
-
+ os.removedirs(self.directory)
+
def test_up_to_date (self):
#
# lists of files
- #
-
+ #
+
self.assertTrue(not task.needs_update_check_directory_missing ([self.directory])[0])
self.assertTrue( task.needs_update_check_directory_missing (["missing directory"])[0])
self.assertRaises(task.error_not_a_directory,
task.needs_update_check_directory_missing, [self.tempfile])
-
-
-
-unittest.main()
+
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/ruffus/test/test_transform_add_inputs.py b/ruffus/test/test_transform_add_inputs.py
index 0fd84fd..e7b9988 100755
--- a/ruffus/test/test_transform_add_inputs.py
+++ b/ruffus/test/test_transform_add_inputs.py
@@ -3,141 +3,48 @@ from __future__ import print_function
"""
test_transform_with_no_re_matches.py
-
- test messages with no regular expression matches
-
-"""
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# options
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-from optparse import OptionParser
-import sys, os
-import os.path
-try:
- import StringIO as io
-except:
- import io as io
-import re,time
-
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-if __name__ == '__main__':
- module_name = os.path.split(sys.argv[0])[1]
- module_name = os.path.splitext(module_name)[0];
-else:
- module_name = __name__
+ test messages with no regular expression matches
+"""
-import ruffus
-print("\tRuffus Version = ", ruffus.__version__)
-parser = OptionParser(version="%%prog v1.0, ruffus v%s" % ruffus.ruffus_version.__version)
-parser.add_option("-t", "--target_tasks", dest="target_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Target task(s) of pipeline.")
-parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Pipeline task(s) which will be included even if they are up to date.")
-parser.add_option("-j", "--jobs", dest="jobs",
- default=1,
- metavar="jobs",
- type="int",
- help="Specifies the number of jobs (commands) to run simultaneously.")
-parser.add_option("-v", "--verbose", dest = "verbose",
- action="count", default=0,
- help="Print more verbose messages for each additional verbose level.")
-parser.add_option("-d", "--dependency", dest="dependency_file",
- #default="simple.svg",
- metavar="FILE",
- type="string",
- help="Print a dependency graph of the pipeline that would be executed "
- "to FILE, but do not execute it.")
-parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
- metavar="FORMAT",
- type="string",
- default = 'svg',
- help="format of dependency graph file. Can be 'ps' (PostScript), "+
- "'svg' 'svgz' (Structured Vector Graphics), " +
- "'png' 'gif' (bitmap graphics) etc ")
-parser.add_option("-n", "--just_print", dest="just_print",
- action="store_true", default=False,
- help="Print a description of the jobs that would be executed, "
- "but do not execute them.")
-parser.add_option("-M", "--minimal_rebuild_mode", dest="minimal_rebuild_mode",
- action="store_true", default=False,
- help="Rebuild a minimum of tasks necessary for the target. "
- "Ignore upstream out of date tasks if intervening tasks are fine.")
-parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
- action="store_true", default=False,
- help="Do not print out legend and key for dependency graph.")
-parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
- action="store_true", default=False,
- help="Draw horizontal dependency graph.")
-
-parameters = [
- ]
-
-
+import os
+import sys
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+for attr in "follows", "transform", "merge", "add_inputs", "inputs", "mkdir", "regex", "pipeline_run", "Pipeline":
+ globals()[attr] = getattr (ruffus, attr)
-# imports
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-import re
-import operator
-import sys,os
-from collections import defaultdict
-import random
-sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
+print("\tRuffus Version = ", ruffus.__version__)
-# use simplejson in place of json for python < 2.6
-try:
- import json
-except ImportError:
- import simplejson
- json = simplejson
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-# Main logic
+# imports
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+import shutil
-
-
-
-# get help string
-f =io.StringIO()
-parser.print_help(f)
-helpstr = f.getvalue()
-(options, remaining_args) = parser.parse_args()
-
-def touch (filename):
- with open(filename, "w"):
+def touch (outfile):
+ with open(outfile, "w"):
pass
@@ -149,23 +56,26 @@ def touch (filename):
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
tempdir = "tempdir/"
@follows(mkdir(tempdir))
- at files([[None, tempdir+ "a.1"], [None, tempdir+ "b.1"]])
-def task1(i, o):
+ at ruffus.files([[None, tempdir+ "a.1"], [None, tempdir+ "b.1"]])
+def task1(i, o):
touch(o)
@follows(mkdir(tempdir))
- at files([[None, tempdir+ "c.1"], [None, tempdir+ "d.1"]])
-def task2(i, o):
+ at ruffus.files([[None, tempdir+ "c.1"], [None, tempdir+ "d.1"]])
+def task2(i, o):
touch(o)
-
- at transform(task1, regex(r"(.*)"), add_inputs(task2, "test_transform_inputs.*"), r"\1.output")
+
+#@transform(input = task1, filter = regex(r"(.*)"), add_inputs = (task2, "test_transform_inputs.*y"), output = r"\1.output")
+ at transform(input = task1, filter = regex(r"(.*)"), add_inputs = add_inputs(task2, "test_transform_inputs.*y"), output = r"\1.output")
+#@transform(input = task1, filter = regex(r"(.*)"), replace_inputs = [task2, "test_transform_inputs.*y"], output = r"\1.output")
+#@transform(input = task1, filter = regex(r"(.*)"), replace_inputs = inputs([task2, "test_transform_inputs.*y"]), output = r"\1.output")
def task3_add_inputs(i, o):
names = ",".join(sorted(i))
with open(o, "w") as oo:
oo.write(names)
-
+
@merge((task3_add_inputs), tempdir + "final.output")
def task4(i, o):
with open(o, "w") as o_file:
@@ -173,22 +83,16 @@ def task4(i, o):
with open(f) as ii:
o_file.write(f +":" + ii.read() + ";")
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
import unittest
class Test_task(unittest.TestCase):
@@ -196,37 +100,45 @@ class Test_task(unittest.TestCase):
def tearDown (self):
"""
"""
- import glob
- for f in glob.glob(tempdir + "*"):
- os.unlink(f)
- os.rmdir(tempdir)
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
def test_task (self):
- pipeline_run([task4], options.forced_tasks, multiprocess = options.jobs,
- verbose = options.verbose)
-
+ pipeline_run(multiprocess = 10, verbose = 0)
+
+ correct_output = "tempdir/a.1.output:tempdir/a.1,tempdir/c.1,tempdir/d.1,test_transform_inputs.py;tempdir/b.1.output:tempdir/b.1,tempdir/c.1,tempdir/d.1,test_transform_inputs.py;"
+ with open(tempdir + "final.output") as real_output:
+ real_output_str = real_output.read()
+ self.assertEqual(correct_output, real_output_str)
+
+
+ def test_newstyle_task (self):
+ test_pipeline = Pipeline("test")
+
+ test_pipeline.files(task1, [[None, tempdir+ "a.1"], [None, tempdir+ "b.1"]])\
+ .follows(mkdir(tempdir))
+ test_pipeline.files(task2, [[None, tempdir+ "c.1"], [None, tempdir+ "d.1"]])\
+ .follows(mkdir(tempdir))
+ test_pipeline.transform(task_func = task3_add_inputs,
+ input = task1,
+ filter = regex(r"(.*)"),
+ add_inputs = add_inputs(task2, "test_transform_inputs.*y"),
+ output = r"\1.output")
+ test_pipeline.merge( task_func = task4,
+ input = task3_add_inputs,
+ output = tempdir + "final.output")
+ test_pipeline.run(multiprocess = 10, verbose = 0)
+
correct_output = "tempdir/a.1.output:tempdir/a.1,tempdir/c.1,tempdir/d.1,test_transform_inputs.py;tempdir/b.1.output:tempdir/b.1,tempdir/c.1,tempdir/d.1,test_transform_inputs.py;"
with open(tempdir + "final.output") as real_output:
real_output_str = real_output.read()
- self.assertTrue(correct_output == real_output_str)
-
+ self.assertEqual(correct_output, real_output_str)
+
+
if __name__ == '__main__':
- if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
- verbose = options.verbose,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
-
- elif options.dependency_file:
- pipeline_printout_graph ( open(options.dependency_file, "w"),
- options.dependency_graph_format,
- options.target_tasks,
- options.forced_tasks,
- draw_vertically = not options.draw_horizontally,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- no_key_legend = options.no_key_legend_in_graph)
- else:
- sys.argv= sys.argv[0:1]
- unittest.main()
+ unittest.main()
diff --git a/ruffus/test/test_transform_inputs.py b/ruffus/test/test_transform_inputs.py
old mode 100644
new mode 100755
index ac2cbaa..611873b
--- a/ruffus/test/test_transform_inputs.py
+++ b/ruffus/test/test_transform_inputs.py
@@ -3,143 +3,48 @@ from __future__ import print_function
"""
test_transform_with_no_re_matches.py
-
- test messages with no regular expression matches
-
-"""
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# options
-
-
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-from optparse import OptionParser
-import sys, os
-import os.path
-try:
- import StringIO as io
-except:
- import io as io
-import re,time
-
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-if __name__ == '__main__':
- module_name = os.path.split(sys.argv[0])[1]
- module_name = os.path.splitext(module_name)[0];
-else:
- module_name = __name__
-
-
-
-import ruffus
-print("\tRuffus Version = ", ruffus.__version__)
-parser = OptionParser(version="%%prog v1.0, ruffus v%s" % ruffus.ruffus_version.__version)
-parser.add_option("-t", "--target_tasks", dest="target_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Target task(s) of pipeline.")
-parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Pipeline task(s) which will be included even if they are up to date.")
-parser.add_option("-j", "--jobs", dest="jobs",
- default=1,
- metavar="jobs",
- type="int",
- help="Specifies the number of jobs (commands) to run simultaneously.")
-parser.add_option("-v", "--verbose", dest = "verbose",
- action="count", default=0,
- help="Print more verbose messages for each additional verbose level.")
-parser.add_option("-d", "--dependency", dest="dependency_file",
- #default="simple.svg",
- metavar="FILE",
- type="string",
- help="Print a dependency graph of the pipeline that would be executed "
- "to FILE, but do not execute it.")
-parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
- metavar="FORMAT",
- type="string",
- default = 'svg',
- help="format of dependency graph file. Can be 'ps' (PostScript), "+
- "'svg' 'svgz' (Structured Vector Graphics), " +
- "'png' 'gif' (bitmap graphics) etc ")
-parser.add_option("-n", "--just_print", dest="just_print",
- action="store_true", default=False,
- help="Print a description of the jobs that would be executed, "
- "but do not execute them.")
-parser.add_option("-M", "--minimal_rebuild_mode", dest="minimal_rebuild_mode",
- action="store_true", default=False,
- help="Rebuild a minimum of tasks necessary for the target. "
- "Ignore upstream out of date tasks if intervening tasks are fine.")
-parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
- action="store_true", default=False,
- help="Do not print out legend and key for dependency graph.")
-parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
- action="store_true", default=False,
- help="Draw horizontal dependency graph.")
-
-parameters = [
- ]
-
-
+ test messages with no regular expression matches
+"""
+import os
+import sys
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
-# imports
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+ruffus = __import__ (ruffus_name)
-import re
-import operator
-import sys,os
-from collections import defaultdict
-import random
+for attr in "follows", "mkdir", "transform", "regex", "merge", "Pipeline", "pipeline_run":
+ globals()[attr] = getattr (ruffus, attr)
-sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-# use simplejson in place of json for python < 2.6
-try:
- import json
-except ImportError:
- import simplejson
- json = simplejson
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-# Main logic
+# imports
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+import shutil
-def touch (filename):
- with open(filename, "w"):
+def touch (outfile):
+ with open(outfile, "w"):
pass
-# get help string
-f =io.StringIO()
-parser.print_help(f)
-helpstr = f.getvalue()
-(options, remaining_args) = parser.parse_args()
-
-
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
# Tasks
@@ -148,24 +53,24 @@ helpstr = f.getvalue()
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
tempdir = "tempdir/"
@follows(mkdir(tempdir))
- at files([[None, tempdir+ "a.1"], [None, tempdir+ "b.1"]])
-def task1(i, o):
+ at ruffus.files([[None, tempdir+ "a.1"], [None, tempdir+ "b.1"]])
+def task1(i, o):
touch(o)
@follows(mkdir(tempdir))
- at files([[None, tempdir+ "c.1"], [None, tempdir+ "d.1"]])
-def task2(i, o):
+ at ruffus.files([[None, tempdir+ "c.1"], [None, tempdir+ "d.1"]])
+def task2(i, o):
touch(o)
-
- at transform(task1, regex(r"(.*)"), inputs(((r"\1"), task2, "test_transform_inputs.*")), r"\1.output")
+
+ at transform(task1, regex(r"(.*)"), ruffus.inputs(((r"\1"), task2, "test_transform_inputs.*y")), r"\1.output")
def task3(i, o):
names = ",".join(sorted(i))
for f in o:
with open(o, "w") as ff:
ff.write(names)
-
+
@merge((task3), tempdir + "final.output")
def task4(i, o):
with open(o, "w") as o_file:
@@ -180,38 +85,46 @@ class Test_task(unittest.TestCase):
def tearDown (self):
"""
"""
- import glob
- for f in glob.glob(tempdir + "*"):
- os.unlink(f)
- os.rmdir(tempdir)
+ try:
+ shutil.rmtree(tempdir)
+ except:
+ pass
def test_task (self):
- pipeline_run([task4], options.forced_tasks, multiprocess = options.jobs,
- verbose = options.verbose)
-
+ pipeline_run([task4], multiprocess = 10, verbose = 0)
+
correct_output = "tempdir/a.1.output:tempdir/a.1,tempdir/c.1,tempdir/d.1,test_transform_inputs.py;tempdir/b.1.output:tempdir/b.1,tempdir/c.1,tempdir/d.1,test_transform_inputs.py;"
with open(tempdir + "final.output") as ff:
real_output = ff.read()
- self.assertTrue(correct_output == real_output)
-
+ self.assertEqual(correct_output, real_output)
+
+ def test_newstyle_task (self):
+ test_pipeline = Pipeline("test")
+
+
+ test_pipeline.files(task1, [[None, tempdir+ "a.1"], [None, tempdir+ "b.1"]])\
+ .follows(mkdir(tempdir))
+
+
+ test_pipeline.files(task2, [[None, tempdir+ "c.1"], [None, tempdir+ "d.1"]])\
+ .follows(mkdir(tempdir))
+
+ test_pipeline.transform(task_func = task3,
+ input = task1,
+ filter = regex(r"(.*)"),
+ replace_inputs = ruffus.inputs(((r"\1"), task2, "test_transform_inputs.*y")),
+ output = r"\1.output")
+ test_pipeline.merge(task4, (task3), tempdir + "final.output")
+
+ test_pipeline.run([task4], multiprocess = 10, verbose = 0)
+
+ correct_output = "tempdir/a.1.output:tempdir/a.1,tempdir/c.1,tempdir/d.1,test_transform_inputs.py;tempdir/b.1.output:tempdir/b.1,tempdir/c.1,tempdir/d.1,test_transform_inputs.py;"
+ with open(tempdir + "final.output") as ff:
+ real_output = ff.read()
+ self.assertEqual(correct_output, real_output)
+
if __name__ == '__main__':
- if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
- verbose = options.verbose,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
-
- elif options.dependency_file:
- with open(options.dependency_file, "w") as graph_printout_file:
- pipeline_printout_graph (graph_printout_file,
- options.dependency_graph_format,
- options.target_tasks,
- options.forced_tasks,
- draw_vertically = not options.draw_horizontally,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- no_key_legend = options.no_key_legend_in_graph)
- else:
- sys.argv= sys.argv[0:1]
- unittest.main()
+ unittest.main()
diff --git a/ruffus/test/test_transform_with_no_re_matches.py b/ruffus/test/test_transform_with_no_re_matches.py
index c0ac59b..ba9ad17 100755
--- a/ruffus/test/test_transform_with_no_re_matches.py
+++ b/ruffus/test/test_transform_with_no_re_matches.py
@@ -3,126 +3,47 @@ from __future__ import print_function
"""
test_transform_with_no_re_matches.py
-
+
test messages with no regular expression matches
-
+
"""
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-# options
+# options
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-from optparse import OptionParser
-import sys, os
-import os.path
-try:
- import StringIO as io
-except:
- import io as io
-
-import re,time
-
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-if __name__ == '__main__':
- module_name = os.path.split(sys.argv[0])[1]
- module_name = os.path.splitext(module_name)[0];
-else:
- module_name = __name__
-
-
-
-import ruffus
-print("\tRuffus Version = ", ruffus.__version__)
-parser = OptionParser(version="%%prog v1.0, ruffus v%s" % ruffus.ruffus_version.__version)
-parser.add_option("-t", "--target_tasks", dest="target_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Target task(s) of pipeline.")
-parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Pipeline task(s) which will be included even if they are up to date.")
-parser.add_option("-j", "--jobs", dest="jobs",
- default=1,
- metavar="jobs",
- type="int",
- help="Specifies the number of jobs (commands) to run simultaneously.")
-parser.add_option("-v", "--verbose", dest = "verbose",
- action="count", default=0,
- help="Print more verbose messages for each additional verbose level.")
-parser.add_option("-d", "--dependency", dest="dependency_file",
- #default="simple.svg",
- metavar="FILE",
- type="string",
- help="Print a dependency graph of the pipeline that would be executed "
- "to FILE, but do not execute it.")
-parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
- metavar="FORMAT",
- type="string",
- default = 'svg',
- help="format of dependency graph file. Can be 'ps' (PostScript), "+
- "'svg' 'svgz' (Structured Vector Graphics), " +
- "'png' 'gif' (bitmap graphics) etc ")
-parser.add_option("-n", "--just_print", dest="just_print",
- action="store_true", default=False,
- help="Print a description of the jobs that would be executed, "
- "but do not execute them.")
-parser.add_option("-M", "--minimal_rebuild_mode", dest="minimal_rebuild_mode",
- action="store_true", default=False,
- help="Rebuild a minimum of tasks necessary for the target. "
- "Ignore upstream out of date tasks if intervening tasks are fine.")
-parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
- action="store_true", default=False,
- help="Do not print out legend and key for dependency graph.")
-parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
- action="store_true", default=False,
- help="Draw horizontal dependency graph.")
-
-parameters = [
- ]
-
+import os
+import sys
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+for attr in "files", "transform", "regex", "pipeline_run", "Pipeline":
+ globals()[attr] = getattr (ruffus, attr)
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-
-# imports
-
+print(" Ruffus Version = ", ruffus.__version__)
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-import re
-import operator
-import sys,os
-from collections import defaultdict
-import random
-sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-# use simplejson in place of json for python < 2.6
-try:
- import json
-except ImportError:
- import simplejson
- json = simplejson
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-# Main logic
+# imports
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -130,14 +51,6 @@ except ImportError:
-
-# get help string
-f =io.StringIO()
-parser.print_help(f)
-helpstr = f.getvalue()
-(options, remaining_args) = parser.parse_args()
-
-
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
# Tasks
@@ -158,63 +71,58 @@ def task_2 (i, o):
import unittest
+
+class t_save_to_str_logger:
+ """
+ Everything to stderr
+ """
+ def __init__ (self):
+ self.info_str = ""
+ self.warning_str = ""
+ self.debug_str = ""
+ def info (self, message):
+ self.info_str += message
+ def warning (self, message):
+ self.warning_str += message
+ def debug (self, message):
+ self.debug_str += message
+
class Test_task_mkdir(unittest.TestCase):
def setUp (self):
"""
"""
pass
-
+
def tearDown (self):
"""
"""
for d in ['a']:
- fullpath = os.path.join(exe_path, d)
+ fullpath = os.path.join(os.path.dirname(__file__), d)
os.unlink(fullpath)
def test_no_re_match (self):
- class t_save_to_str_logger:
- """
- Everything to stderr
- """
- def __init__ (self):
- self.info_str = ""
- self.warning_str = ""
- self.debug_str = ""
- def info (self, message):
- self.info_str += message
- def warning (self, message):
- self.warning_str += message
- def debug (self, message):
- self.debug_str += message
save_to_str_logger = t_save_to_str_logger()
- pipeline_run([task_2], options.forced_tasks, multiprocess = options.jobs,
- logger = save_to_str_logger,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- verbose = 1)
+ pipeline_run(multiprocess = 10, logger = save_to_str_logger, verbose = 1)
+
+ self.assertTrue("no files names matched" in save_to_str_logger.warning_str)
+ print("\n Warning printed out correctly", file=sys.stderr)
+
+ def test_newstyle_no_re_match (self):
+
+ test_pipeline = Pipeline("test")
+ test_pipeline.files(task_1, None, "a")
+ test_pipeline.transform(task_2, task_1, regex("b"), "task_2.output")
+
+ save_to_str_logger = t_save_to_str_logger()
+ test_pipeline.run(multiprocess = 10, logger = save_to_str_logger, verbose = 1)
self.assertTrue("no files names matched" in save_to_str_logger.warning_str)
print("\n Warning printed out correctly", file=sys.stderr)
-
+
if __name__ == '__main__':
- if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
- verbose = options.verbose,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
-
- elif options.dependency_file:
- with open(options.dependency_file, "w") as graph_file:
- pipeline_printout_graph ( graph_file,
- options.dependency_graph_format,
- options.target_tasks,
- options.forced_tasks,
- draw_vertically = not options.draw_horizontally,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- no_key_legend = options.no_key_legend_in_graph)
- else:
- sys.argv= sys.argv[0:1]
- unittest.main()
+ unittest.main()
diff --git a/ruffus/test/test_tutorial7.py b/ruffus/test/test_tutorial7.py
index e42362d..02d660b 100755
--- a/ruffus/test/test_tutorial7.py
+++ b/ruffus/test/test_tutorial7.py
@@ -1,10 +1,5 @@
#!/usr/bin/env python
from __future__ import print_function
-# make sure using local ruffus
-import sys, os
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0, os.path.abspath(os.path.join(exe_path,"..", "..")))
-
NUMBER_OF_RANDOMS = 10000
@@ -13,8 +8,24 @@ working_dir = "temp_tutorial7/"
-import time, sys, os
-from ruffus import *
+
+import os
+import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+for attr in "follows", "split", "mkdir", "files", "transform", "suffix", "posttask", "touch_file", "merge", "Pipeline":
+ globals()[attr] = getattr (ruffus, attr)
import random
import glob
@@ -30,6 +41,7 @@ def create_random_numbers(input_file_name, output_file_name):
f = open(output_file_name, "w")
for i in range(NUMBER_OF_RANDOMS):
f.write("%g\n" % (random.random() * 100.0))
+ f.close()
#---------------------------------------------------------------
#
@@ -52,11 +64,16 @@ def step_4_split_numbers_into_chunks (input_file_name, output_files):
#
output_file = None
cnt_files = 0
- for i, line in enumerate(open(input_file_name)):
- if i % CHUNK_SIZE == 0:
- cnt_files += 1
- output_file = open(working_dir + "%d.chunks" % cnt_files, "w")
- output_file.write(line)
+ with open(input_file_name) as ii:
+ for i, line in enumerate(ii):
+ if i % CHUNK_SIZE == 0:
+ cnt_files += 1
+ if output_file:
+ output_file.close()
+ output_file = open(working_dir + "%d.chunks" % cnt_files, "w")
+ output_file.write(line)
+ if output_file:
+ output_file.close()
#---------------------------------------------------------------
#
@@ -64,29 +81,30 @@ def step_4_split_numbers_into_chunks (input_file_name, output_files):
#
@transform(step_4_split_numbers_into_chunks, suffix(".chunks"), ".sums")
def step_5_calculate_sum_of_squares (input_file_name, output_file_name):
- output = open(output_file_name, "w")
- sum_squared, sum = [0.0, 0.0]
- cnt_values = 0
- for line in open(input_file_name):
- cnt_values += 1
- val = float(line.rstrip())
- sum_squared += val * val
- sum += val
- output.write("%s\n%s\n%d\n" % (repr(sum_squared), repr(sum), cnt_values))
+ with open(output_file_name, "w") as oo:
+ sum_squared, sum = [0.0, 0.0]
+ cnt_values = 0
+ with open(input_file_name) as ii:
+ for line in ii:
+ cnt_values += 1
+ val = float(line.rstrip())
+ sum_squared += val * val
+ sum += val
+ oo.write("%s\n%s\n%d\n" % (repr(sum_squared), repr(sum), cnt_values))
def print_hooray_again():
- print("hooray again")
+ print(" hooray again")
def print_whoppee_again():
- print("whoppee again")
+ print(" whoppee again")
#---------------------------------------------------------------
#
# Calculate sum and sum of squares for each chunk
#
- at posttask(lambda: sys.stdout.write("hooray\n"))
+ at posttask(lambda: sys.stdout.write(" hooray\n"))
@posttask(print_hooray_again, print_whoppee_again, touch_file(os.path.join(working_dir, "done")))
@merge(step_5_calculate_sum_of_squares, os.path.join(working_dir, "variance.result"))
def step_6_calculate_variance (input_file_names, output_file_name):
@@ -104,7 +122,8 @@ def step_6_calculate_variance (input_file_names, output_file_name):
# added up all the sum_squared, and sum and cnt_values from all the chunks
#
for input_file_name in input_file_names:
- sum_squared, sum, cnt_values = list(map(float, open(input_file_name).readlines()))
+ with open(input_file_name) as ii:
+ sum_squared, sum, cnt_values = list(map(float, ii.readlines()))
all_sum_squared += sum_squared
all_sum += sum
all_cnt_values += cnt_values
@@ -114,11 +133,65 @@ def step_6_calculate_variance (input_file_names, output_file_name):
# print output
#
print(variance, file=output)
+ output.close()
+
+
+import unittest, shutil
+try:
+ from StringIO import StringIO
+except:
+ from io import StringIO
+
+class Test_ruffus(unittest.TestCase):
+ def setUp(self):
+ try:
+ shutil.rmtree(working_dir)
+ except:
+ pass
+
+ def tearDown(self):
+ try:
+ shutil.rmtree(working_dir)
+ pass
+ except:
+ pass
+
+ def atest_ruffus (self):
+ pipeline_run(multiprocess = 50, verbose = 0)
+ output_file = os.path.join(working_dir, "variance.result")
+ if not os.path.exists (output_file):
+ raise Exception("Missing %s" % output_file)
+
+
+ def test_newstyle_ruffus (self):
+ test_pipeline = Pipeline("test")
+
+ test_pipeline.files(create_random_numbers, None, working_dir + "random_numbers.list")\
+ .follows(mkdir(working_dir))
+
+
+ test_pipeline.split(task_func = step_4_split_numbers_into_chunks,
+ input = working_dir + "random_numbers.list",
+ output = working_dir + "*.chunks")\
+ .follows(create_random_numbers)
+
+ test_pipeline.transform(task_func = step_5_calculate_sum_of_squares,
+ input = step_4_split_numbers_into_chunks,
+ filter = suffix(".chunks"),
+ output = ".sums")
+
+ test_pipeline.merge(task_func = step_6_calculate_variance, input = step_5_calculate_sum_of_squares, output = os.path.join(working_dir, "variance.result"))\
+ .posttask(lambda: sys.stdout.write(" hooray\n"))\
+ .posttask(print_hooray_again, print_whoppee_again, touch_file(os.path.join(working_dir, "done")))
+
+ test_pipeline.run(multiprocess = 50, verbose = 0)
+ output_file = os.path.join(working_dir, "variance.result")
+ if not os.path.exists (output_file):
+ raise Exception("Missing %s" % output_file)
+
+
+
+if __name__ == '__main__':
+ unittest.main()
+
-#---------------------------------------------------------------
-#
-# Run
-#
-pipeline_run([step_6_calculate_variance], verbose = 1)
-import shutil
-shutil.rmtree(working_dir)
diff --git a/ruffus/test/test_unicode_filenames.py b/ruffus/test/test_unicode_filenames.py
index 29778c5..73e3dd7 100755
--- a/ruffus/test/test_unicode_filenames.py
+++ b/ruffus/test/test_unicode_filenames.py
@@ -3,97 +3,41 @@ from __future__ import print_function
"""
test_follows_mkdir.py
-
+
test make directory dependencies
-
+
use :
-j N / --jobs N to speify multitasking
-v to see the jobs in action
- -n / --just_print to see what jobs would run
+ -n / --just_print to see what jobs would run
"""
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
+import os
+import sys
-# options
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
-#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-from optparse import OptionParser
-import sys, os
-import os.path
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = __import__ (ruffus_name)
+
try:
- import StringIO as io
-except:
- import io as io
-import re,time
-
-# add self to search path for testing
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
-if __name__ == '__main__':
- module_name = os.path.split(sys.argv[0])[1]
- module_name = os.path.splitext(module_name)[0];
-else:
- module_name = __name__
-
-
-
-import ruffus
-print(ruffus.__version__)
-parser = OptionParser(version="%%prog v1.0, ruffus v%s" % ruffus.ruffus_version.__version)
-parser.add_option("-t", "--target_tasks", dest="target_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Target task(s) of pipeline.")
-parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
- action="append",
- default = list(),
- metavar="JOBNAME",
- type="string",
- help="Pipeline task(s) which will be included even if they are up to date.")
-parser.add_option("-j", "--jobs", dest="jobs",
- default=1,
- metavar="jobs",
- type="int",
- help="Specifies the number of jobs (commands) to run simultaneously.")
-parser.add_option("-v", "--verbose", dest = "verbose",
- action="count", default=0,
- help="Print more verbose messages for each additional verbose level.")
-parser.add_option("-d", "--dependency", dest="dependency_file",
- #default="simple.svg",
- metavar="FILE",
- type="string",
- help="Print a dependency graph of the pipeline that would be executed "
- "to FILE, but do not execute it.")
-parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
- metavar="FORMAT",
- type="string",
- default = 'svg',
- help="format of dependency graph file. Can be 'ps' (PostScript), "+
- "'svg' 'svgz' (Structured Vector Graphics), " +
- "'png' 'gif' (bitmap graphics) etc ")
-parser.add_option("-n", "--just_print", dest="just_print",
- action="store_true", default=False,
- help="Print a description of the jobs that would be executed, "
- "but do not execute them.")
-parser.add_option("-M", "--minimal_rebuild_mode", dest="minimal_rebuild_mode",
- action="store_true", default=False,
- help="Rebuild a minimum of tasks necessary for the target. "
- "Ignore upstream out of date tasks if intervening tasks are fine.")
-parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
- action="store_true", default=False,
- help="Do not print out legend and key for dependency graph.")
-parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
- action="store_true", default=False,
- help="Draw horizontal dependency graph.")
-
-parameters = [
- ]
+ attrlist = ruffus.__all__
+except AttributeError:
+ attrlist = dir (ruffus)
+for attr in attrlist:
+ if attr[0:2] != "__":
+ globals()[attr] = getattr (ruffus, attr)
+
@@ -103,26 +47,18 @@ parameters = [
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-# imports
+# imports
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-import re
-import operator
-import sys,os
-from collections import defaultdict
-import random
-
-sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-
+import json
# use simplejson in place of json for python < 2.6
-try:
- import json
-except ImportError:
- import simplejson
- json = simplejson
+#try:
+# import json
+#except ImportError:
+# import simplejson
+# json = simplejson
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
@@ -135,11 +71,6 @@ except ImportError:
-# get help string
-f =io.StringIO()
-parser.print_help(f)
-helpstr = f.getvalue()
-(options, remaining_args) = parser.parse_args()
def touch (filename):
with open(filename, "w"):
@@ -154,7 +85,7 @@ if sys.hexversion >= 0x03000000:
#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
-directories = [os.path.abspath(unicode("a")), unicode("b")]
+directories = [os.path.abspath(unicode("a")), unicode("b")]
@follows(mkdir(directories), mkdir(unicode("c")), mkdir(unicode("d"), unicode("e")), mkdir(unicode("e")))
@posttask(touch_file(unicode("f")))
def task_which_makes_directories ():
@@ -173,45 +104,51 @@ class Test_task_mkdir(unittest.TestCase):
"""
"""
pass
-
+
def tearDown (self):
"""
delete directories
"""
for d in 'abcde':
- fullpath = os.path.join(exe_path, d)
+ fullpath = os.path.join(os.path.dirname(__file__), d)
os.rmdir(fullpath)
for d in 'fgh':
- fullpath = os.path.join(exe_path, d)
+ fullpath = os.path.join(os.path.dirname(__file__), d)
os.unlink(fullpath)
def test_mkdir (self):
- pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
- logger = stderr_logger if options.verbose else black_hole_logger,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- verbose = options.verbose)
-
+ pipeline_run(multiprocess = 10, verbose = 0)
+
for d in 'abcdefgh':
- fullpath = os.path.join(exe_path, d)
+ fullpath = os.path.join(os.path.dirname(__file__), d)
self.assertTrue(os.path.exists(fullpath))
+ def test_newstyle_mkdir (self):
+ test_pipeline = Pipeline("test")
+
+ test_pipeline.follows(task_which_makes_directories,
+ mkdir(directories),
+ mkdir(unicode("c")),
+ mkdir(unicode("d"),
+ unicode("e")),
+ mkdir(unicode("e")))\
+ .posttask(touch_file(unicode("f")))
+
+ test_pipeline.files(task_which_makes_files, None, ["g", "h"])
+ test_pipeline.run(multiprocess = 10, verbose = 0)
+
+ for d in 'abcdefgh':
+ fullpath = os.path.join(os.path.dirname(__file__), d)
+ self.assertTrue(os.path.exists(fullpath))
+
+
+
+
+
if __name__ == '__main__':
- if options.just_print:
- pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
- verbose = options.verbose,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
-
- elif options.dependency_file:
- pipeline_printout_graph ( open(options.dependency_file, "w"),
- options.dependency_graph_format,
- options.target_tasks,
- options.forced_tasks,
- draw_vertically = not options.draw_horizontally,
- gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
- no_key_legend = options.no_key_legend_in_graph)
- else:
- sys.argv= sys.argv[0:1]
- unittest.main()
+ unittest.main()
+
+
diff --git a/ruffus/test/test_verbosity.py b/ruffus/test/test_verbosity.py
index b52e21a..55115e9 100755
--- a/ruffus/test/test_verbosity.py
+++ b/ruffus/test/test_verbosity.py
@@ -5,42 +5,51 @@ from __future__ import print_function
test_verbosity.py
"""
-
+temp_dir = "test_verbosity/"
import unittest
+
import os
import sys
+
+# add grandparent to search path for testing
+grandparent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.insert(0, grandparent_dir)
+
+# module name = script name without extension
+module_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+# funky code to import by file name
+parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ruffus_name = os.path.basename(parent_dir)
+ruffus = list(map(__import__, [ruffus_name]))[0]
+
+
import shutil
try:
from StringIO import StringIO
except:
from io import StringIO
-
-import time
import re
-exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
-sys.path.insert(0, os.path.abspath(os.path.join(exe_path,"..", "..")))
-from ruffus import *
-from ruffus import (pipeline_run, pipeline_printout, suffix, transform, split,
- merge, dbdict, follows)
-from ruffus.ruffus_exceptions import RethrownJobError
-from ruffus.ruffus_utility import (RUFFUS_HISTORY_FILE,
- CHECKSUM_FILE_TIMESTAMPS)
+ruffus = __import__ (ruffus_name)
+for attr in "pipeline_run", "pipeline_printout", "suffix", "transform", "split", "merge", "dbdict", "follows", "mkdir", "originate", "Pipeline":
+ globals()[attr] = getattr (ruffus, attr)
+RethrownJobError = ruffus.ruffus_exceptions.RethrownJobError
+RUFFUS_HISTORY_FILE = ruffus.ruffus_utility.RUFFUS_HISTORY_FILE
+CHECKSUM_FILE_TIMESTAMPS = ruffus.ruffus_utility.CHECKSUM_FILE_TIMESTAMPS
-parser = cmdline.get_argparse(description='WHAT DOES THIS PIPELINE DO?')
-options = parser.parse_args()
-logger, logger_mutex = cmdline.setup_logging (__name__, options.log_file, options.verbose)
-
#---------------------------------------------------------------
# create initial files
#
- at originate([ ['/data/scratch/lg/what/one/two/three/job1.a.start', 'job1.b.start'],
- ['/data/scratch/lg/what/one/two/three/job2.a.start', 'job2.b.start'],
- ['/data/scratch/lg/what/one/two/three/job3.a.start', 'job3.b.start'] ])
+ at mkdir(temp_dir + 'data/scratch/lg/what/one/two/three/')
+ at originate([ [temp_dir + 'data/scratch/lg/what/one/two/three/job1.a.start', temp_dir + 'job1.b.start'],
+ [temp_dir + 'data/scratch/lg/what/one/two/three/job2.a.start', temp_dir + 'job2.b.start'],
+ [temp_dir + 'data/scratch/lg/what/one/two/three/job3.a.start', temp_dir + 'job3.b.start'] ])
def create_initial_file_pairs(output_files):
# create both files as necessary
for output_file in output_files:
@@ -59,6 +68,18 @@ def first_task(input_files, output_file):
def second_task(input_files, output_file):
with open(output_file, "w"): pass
+test_pipeline = Pipeline("test")
+test_pipeline.originate(output = [ [temp_dir + 'data/scratch/lg/what/one/two/three/job1.a.start', temp_dir + 'job1.b.start'],
+ [temp_dir + 'data/scratch/lg/what/one/two/three/job2.a.start', temp_dir + 'job2.b.start'],
+ [temp_dir + 'data/scratch/lg/what/one/two/three/job3.a.start', temp_dir + 'job3.b.start'] ],
+ task_func = create_initial_file_pairs)
+test_pipeline.transform(task_func = first_task, input = create_initial_file_pairs, filter = suffix(".start"), output = ".output.1")
+test_pipeline.transform(input = first_task, filter = suffix(".output.1"), output = ".output.2", task_func= second_task)
+
+
+decorator_syntax = 0
+oop_syntax = 1
+
class Test_verbosity(unittest.TestCase):
#___________________________________________________________________________
#
@@ -66,10 +87,16 @@ class Test_verbosity(unittest.TestCase):
#___________________________________________________________________________
def test_printout_abbreviated_path1(self):
"""Input file exists, output doesn't exist"""
- s = StringIO()
- pipeline_printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 1)
- self.assertTrue(re.search('Job needs update: Missing files\n\s+'
- '\[\.\.\./job2\.a\.start, job2\.b\.start, \.\.\./job2.a.output.1\]', s.getvalue()))
+ for syntax in decorator_syntax, oop_syntax:
+ s = StringIO()
+ if syntax == oop_syntax:
+ test_pipeline.printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 1)
+ else:
+ pipeline_printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 1, wrap_width = 500)
+ ret = s.getvalue()
+ self.assertTrue(re.search('Job needs update:.*Missing files.*'
+ '\[\.\.\./job2\.a\.start, test_verbosity/job2\.b\.start, \.\.\./job2.a.output.1\]', ret, re.DOTALL) is not None)
+
#___________________________________________________________________________
#
@@ -77,20 +104,30 @@ class Test_verbosity(unittest.TestCase):
#___________________________________________________________________________
def test_printout_abbreviated_path2(self):
"""Input file exists, output doesn't exist"""
- s = StringIO()
- pipeline_printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 2)
- self.assertTrue('[.../three/job1.a.start, job1.b.start, .../three/job1.a.output.1]' in s.getvalue())
+ for syntax in decorator_syntax, oop_syntax:
+ s = StringIO()
+ if syntax == oop_syntax:
+ test_pipeline.printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 2, wrap_width = 500)
+ else:
+ pipeline_printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 2, wrap_width = 500)
+ ret = s.getvalue()
+ self.assertTrue('[.../three/job1.a.start, test_verbosity/job1.b.start, .../three/job1.a.output.1]' in ret)
#___________________________________________________________________________
#
- # test_printout_abbreviated_path2
+ # test_printout_abbreviated_path3
#___________________________________________________________________________
def test_printout_abbreviated_path3(self):
"""Input file exists, output doesn't exist"""
- s = StringIO()
- pipeline_printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 3)
- self.assertTrue('[.../two/three/job1.a.start, job1.b.start, .../two/three/job1.a.output.1]' in s.getvalue())
+ for syntax in decorator_syntax, oop_syntax:
+ s = StringIO()
+ if syntax == oop_syntax:
+ test_pipeline.printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 3, wrap_width = 500)
+ else:
+ pipeline_printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 3, wrap_width = 500)
+ ret = s.getvalue()
+ self.assertTrue('[.../two/three/job1.a.start, test_verbosity/job1.b.start, .../two/three/job1.a.output.1]' in s.getvalue())
#___________________________________________________________________________
#
@@ -98,10 +135,14 @@ class Test_verbosity(unittest.TestCase):
#___________________________________________________________________________
def test_printout_abbreviated_path9(self):
"""Input file exists, output doesn't exist"""
- s = StringIO()
- pipeline_printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 9)
- ret = s.getvalue()
- self.assertTrue('[/data/scratch/lg/what/one/two/three/job2.a.start, job2.b.start,' in ret)
+ for syntax in decorator_syntax, oop_syntax:
+ s = StringIO()
+ if syntax == oop_syntax:
+ test_pipeline.printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 9, wrap_width = 500)
+ else:
+ pipeline_printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 9, wrap_width = 500)
+ ret = s.getvalue()
+ self.assertTrue('[%sdata/scratch/lg/what/one/two/three/job2.a.start, test_verbosity/job2.b.start,' % temp_dir in ret)
#___________________________________________________________________________
@@ -110,11 +151,18 @@ class Test_verbosity(unittest.TestCase):
#___________________________________________________________________________
def test_printout_abbreviated_path0(self):
"""Input file exists, output doesn't exist"""
- s = StringIO()
- pipeline_printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 0)
- ret = s.getvalue()
- self.assertTrue('[[/data/scratch/lg/what/one/two/three/job2.a.start,' in ret)
- self.assertTrue('/ruffus/test/job2.b.start]' in ret)
+ for syntax in decorator_syntax, oop_syntax:
+ s = StringIO()
+ if syntax == oop_syntax:
+ test_pipeline.printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 0, wrap_width = 500)
+ else:
+ pipeline_printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 0, wrap_width = 500)
+ ret = s.getvalue()
+ path_str = os.path.abspath('%sdata/scratch/lg/what/one/two/three/job2.a.start' % temp_dir)
+ path_str = '[[%s' % path_str
+ self.assertTrue(path_str in ret)
+ self.assertTrue(temp_dir + 'job2.b.start]' in ret)
+
#___________________________________________________________________________
@@ -123,10 +171,15 @@ class Test_verbosity(unittest.TestCase):
#___________________________________________________________________________
def test_printout_abbreviated_path_minus_60(self):
"""Input file exists, output doesn't exist"""
- s = StringIO()
- pipeline_printout(s, [second_task], verbose = 5, verbose_abbreviated_path = -60)
- ret = s.getvalue()
- self.assertTrue('[<???> ratch/lg/what/one/two/three/job2.a.start, job2.b.start]' in ret)
+ for syntax in decorator_syntax, oop_syntax:
+ s = StringIO()
+ if syntax == oop_syntax:
+ test_pipeline.printout(s, [second_task], verbose = 5, verbose_abbreviated_path = -60, wrap_width = 500)
+ else:
+ pipeline_printout(s, [second_task], verbose = 5, verbose_abbreviated_path = -60, wrap_width = 500)
+ ret = s.getvalue()
+ self.assertTrue('[<???> ne/two/three/job2.a.start, test_verbosity/job2.b.start]' in ret)
+
#
# Necessary to protect the "entry point" of the program under windows.
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/python-ruffus.git
More information about the debian-med-commit
mailing list