[hypre] 01/09: New upstream version 2.11.2

Drew Parsons dparsons at moszumanska.debian.org
Fri Sep 29 06:41:36 UTC 2017


This is an automated email from the git hooks/post-receive script.

dparsons pushed a commit to tag experimental/2.11.2-1
in repository hypre.

commit a93b8b781ad0078195cbe82074f5436f75eaa9a8
Author: Drew Parsons <dparsons at emerall.com>
Date:   Fri Sep 29 11:53:09 2017 +0800

    New upstream version 2.11.2
---
 AUTOTEST/cmaketest.sh                              |    2 +-
 AUTOTEST/machine-tux.sh                            |   26 +-
 AUTOTEST/make.filters                              |    1 +
 CHANGELOG                                          |   10 +
 docs/HYPRE_ref_manual.pdf                          |  Bin 872663 -> 874193 bytes
 docs/HYPRE_usr_manual.pdf                          |  Bin 653473 -> 653348 bytes
 src/CMakeLists.txt                                 |    8 +-
 src/FEI_mv/SuperLU/SRC/dgsequ.c                    |    2 +
 src/FEI_mv/SuperLU/SRC/dgssvx.c                    |    2 +
 src/FEI_mv/SuperLU/SRC/dlaqgs.c                    |    2 +
 src/FEI_mv/SuperLU/SRC/dpivotgrowth.c              |    2 +
 src/FEI_mv/fei-hypre/HYPRE_FEI_includes.h          |    7 +
 src/FEI_mv/fei-hypre/HYPRE_LinSysCore.cxx          |    3 +-
 src/FEI_mv/femli/mli_utils.c                       |    2 +
 src/IJ_mv/IJMatrix_parcsr.c                        |    7 +-
 src/Makefile                                       |   34 +-
 src/config/HYPRE_config.h.cmake.in                 |    3 +
 src/config/HYPRE_config.h.in                       |    3 +
 src/config/Makefile.config.in                      |   26 +-
 src/config/config.guess                            |  884 ++--
 src/config/config.sub                              |  478 +-
 src/config/configure.in                            |   69 +-
 src/config/hypre_blas_macros.m4                    |    6 +-
 src/config/hypre_macros_misc.m4                    |   16 +-
 src/configure                                      |  134 +-
 src/distributed_ls/Euclid/Factor_dh.c              |   23 +-
 src/distributed_ls/ParaSails/DiagScale.c           |    2 +-
 src/distributed_ls/ParaSails/Numbering.c           |    2 +-
 src/distributed_ls/ParaSails/OrderStat.c           |    4 +-
 src/distributed_ls/ParaSails/OrderStat.h           |    2 +-
 src/distributed_ls/ParaSails/ParaSails.c           |    2 +-
 src/docs/usr_solvers.tex                           |   10 +-
 src/examples/README_files/ex10.cxx.html            |    4 +-
 src/examples/README_files/ex11.c.html              |  232 +-
 src/examples/README_files/ex12.c.html              |   77 +-
 src/examples/README_files/ex12f.f.html             |   55 +-
 src/examples/README_files/ex13.c.html              |    2 +-
 src/examples/README_files/ex14.c.html              |    2 +-
 src/examples/README_files/ex5.c.html               |  593 +--
 src/examples/README_files/ex5big.c.html            |  523 +-
 src/examples/README_files/ex5f.f.html              |  428 +-
 src/examples/README_files/ex9.c.html               |  292 +-
 src/examples/ex10.cxx                              |    4 +-
 src/examples/ex11.c                                |    4 +-
 src/examples/ex12.c                                |    1 -
 src/examples/ex12f.f                               |    1 +
 src/examples/ex13.c                                |    2 +-
 src/examples/ex14.c                                |    2 +-
 src/examples/ex5.c                                 |    5 +-
 src/examples/ex5big.c                              |    5 +-
 src/examples/ex5f.f                                |    8 +-
 src/examples/ex9.c                                 |    2 +
 src/krylov/lgmres.c                                |    9 +-
 src/parcsr_block_mv/par_block_nodal_systems.c      |    2 +-
 src/parcsr_ls/F90_HYPRE_parcsr_amg.c               |   46 +
 src/parcsr_ls/HYPRE_parcsr_amg.c                   |   34 +
 src/parcsr_ls/HYPRE_parcsr_ls.h                    |   25 +
 src/parcsr_ls/_hypre_parcsr_ls.h                   |   11 +-
 src/parcsr_ls/amg_hybrid.c                         |    2 +-
 src/parcsr_ls/ams.c                                |    2 +-
 src/parcsr_ls/par_add_cycle.c                      |   38 +-
 src/parcsr_ls/par_amg.c                            |   91 +-
 src/parcsr_ls/par_amg.h                            |    6 +-
 src/parcsr_ls/par_amg_setup.c                      |   40 +-
 src/parcsr_ls/par_amg_solve.c                      |    8 +-
 src/parcsr_ls/par_amg_solveT.c                     |   17 +-
 src/parcsr_ls/par_coarsen.c                        |    4 +-
 src/parcsr_ls/par_cycle.c                          |  152 +-
 src/parcsr_ls/par_laplace.c                        |    4 +-
 src/parcsr_ls/par_nodal_systems.c                  |    2 +-
 src/parcsr_ls/par_rap.c                            |    2 +-
 src/parcsr_ls/par_relax.c                          |   22 +-
 src/parcsr_ls/par_rotate_7pt.c                     |    1 +
 src/parcsr_ls/par_schwarz.c                        |   45 +-
 src/parcsr_ls/par_stats.c                          |   42 +-
 src/parcsr_ls/par_strength.c                       |    2 +-
 src/parcsr_ls/schwarz.c                            |   22 +-
 src/parcsr_mv/par_csr_matrix.c                     |    2 +-
 src/seq_mv/csr_matvec.c                            |    2 +-
 src/seq_mv/vector.c                                |    2 +-
 src/sstruct_ls/fac_CFInterfaceExtents.c            |    2 +-
 src/sstruct_ls/fac_amr_fcoarsen.c                  |    4 +-
 src/sstruct_ls/fac_amr_zero_data.c                 |    6 +-
 src/sstruct_ls/fac_cf_coarsen.c                    |    2 +-
 src/sstruct_ls/fac_cfstencil_box.c                 |    2 +-
 src/sstruct_ls/fac_zero_stencilcoef.c              |    2 +-
 src/sstruct_mv/HYPRE_sstruct_grid.c                |   12 +-
 src/sstruct_mv/_hypre_sstruct_mv.h                 |    2 +
 src/sstruct_mv/sstruct_grid.c                      |   26 +-
 src/sstruct_mv/sstruct_grid.h                      |    2 +
 src/sstruct_mv/sstruct_matrix.c                    |   20 +-
 src/sstruct_mv/sstruct_vector.c                    |    4 -
 src/struct_ls/Makefile                             |    2 +-
 src/struct_ls/pfmg.c                               |    4 +
 src/struct_ls/pfmg_setup.c                         |    3 +
 src/struct_ls/pfmg_solve.c                         |    7 +
 src/struct_ls/red_black_gs.c                       |    4 -
 src/struct_ls/smg.c                                |    4 +
 src/struct_ls/smg_setup.c                          |    4 +
 src/struct_ls/smg_solve.c                          |    6 +
 src/struct_mv/assumed_part.c                       |   23 +-
 src/struct_mv/struct_grid.c                        |    7 +-
 src/test/CMakeLists.txt                            |    3 -
 src/test/Makefile                                  |   73 +-
 src/test/TEST_ams/solvers.saved                    |  138 +-
 src/test/TEST_examples/bigint.saved                |   24 +-
 src/test/TEST_examples/default.saved               |  229 +-
 src/test/TEST_ij/agg_interp.jobs                   |   28 +-
 src/test/TEST_ij/agg_interp.saved                  |   38 +-
 src/test/TEST_ij/coarsening.jobs                   |   38 +-
 src/test/TEST_ij/coarsening.saved                  |   54 +-
 src/test/TEST_ij/coarsening.sh                     |    4 +
 src/test/TEST_ij/default.jobs                      |    8 +-
 src/test/TEST_ij/default.saved                     |   24 +-
 src/test/TEST_ij/elast.jobs                        |   40 +-
 src/test/TEST_ij/elast.saved                       |  105 +-
 src/test/TEST_ij/elast.sh                          |    3 +
 src/test/TEST_ij/interp.jobs                       |   18 +-
 src/test/TEST_ij/interp.saved                      |   55 +-
 src/test/TEST_ij/interp.sh                         |    1 +
 src/test/TEST_ij/lobpcg.jobs                       |   36 +
 src/test/TEST_ij/lobpcg.saved                      |  114 +-
 src/test/TEST_ij/lobpcg.sh                         |   19 +
 src/test/TEST_ij/matrix.jobs                       |   20 +-
 src/test/TEST_ij/matrix.saved                      |   32 +-
 src/test/TEST_ij/options.saved                     |   56 +-
 src/test/TEST_ij/schw.00000                        |  231 +
 src/test/TEST_ij/schw.00001                        |    1 +
 src/test/TEST_ij/schw.00002                        |  231 +
 src/test/TEST_ij/smoother.jobs                     |   56 +-
 src/test/TEST_ij/smoother.saved                    |   78 +-
 src/test/TEST_ij/smoother.sh                       |    8 +
 src/test/TEST_ij/solvers.jobs                      |   75 +-
 src/test/TEST_ij/solvers.saved                     |  188 +-
 src/test/TEST_ij/solvers.sh                        |   27 +-
 src/test/TEST_sstruct/addtovalues.saved            |   16 +-
 src/test/TEST_sstruct/amr2d.saved                  |   24 +-
 src/test/TEST_sstruct/emptyProc.saved              |   12 +-
 src/test/TEST_sstruct/maxwell.saved                |    6 +-
 src/test/TEST_sstruct/miller.jobs                  |    6 +-
 src/test/TEST_sstruct/miller.saved                 |   20 +-
 src/test/TEST_sstruct/neumann.jobs                 |   36 +-
 src/test/TEST_sstruct/neumann.saved                |   32 +
 src/test/TEST_sstruct/neumann.sh                   |   28 +-
 src/test/TEST_sstruct/sharedpart.saved             |   14 +-
 src/test/TEST_sstruct/solvers.saved                |   10 +-
 src/test/TEST_sstruct/solvers.sh                   |    9 +-
 src/test/TEST_sstruct/sstruct.in.dirichlet         |   65 +
 src/test/TEST_sstruct/sstruct.in.neumann-yz        |   90 +
 src/test/TEST_sstruct/sstruct.in.wide.2D           |   68 +
 .../lobpcg.jobs => TEST_sstruct/wide.jobs}         |   15 +-
 src/test/TEST_sstruct/wide.saved                   |    8 +
 src/test/TEST_sstruct/{neumann.sh => wide.sh}      |   18 +-
 src/test/TEST_struct/pfmgvcycle.sh                 |   10 +-
 src/test/TEST_struct/solvers.sh                    |    9 +-
 src/test/checktest.sh                              |   64 +
 src/test/cleantest.sh                              |   31 +-
 src/test/ij.c                                      | 5077 ++++++++++++++------
 src/test/runcheck.sh                               |   89 +
 src/test/runtest.sh                                |  126 +-
 src/test/runtest.valgrind                          |    0
 src/test/sstruct.c                                 |   34 +-
 src/test/sstruct.in.README                         |    3 +
 src/test/struct.c                                  |   84 +-
 src/utilities/_hypre_utilities.h                   |   46 +-
 src/utilities/amg_linklist.c                       |   26 +-
 src/utilities/amg_linklist.h                       |    3 -
 .../caliper_instrumentation.h}                     |   28 +-
 src/utilities/general.h                            |    4 +
 169 files changed, 8468 insertions(+), 4495 deletions(-)

diff --git a/AUTOTEST/cmaketest.sh b/AUTOTEST/cmaketest.sh
index a1c8225..699be44 100755
--- a/AUTOTEST/cmaketest.sh
+++ b/AUTOTEST/cmaketest.sh
@@ -13,7 +13,7 @@
 
 testname=`basename $0 .sh`
 
-drivers="ij new_ij sstruct struct ams_driver maxwell_unscaled sstruct_fac ij_mv struct_migrate"
+drivers="ij sstruct struct ams_driver maxwell_unscaled sstruct_fac ij_mv struct_migrate"
 
 # Echo usage information
 case $1 in
diff --git a/AUTOTEST/machine-tux.sh b/AUTOTEST/machine-tux.sh
index 8876a65..a2c82f1 100755
--- a/AUTOTEST/machine-tux.sh
+++ b/AUTOTEST/machine-tux.sh
@@ -65,15 +65,23 @@ co="--enable-debug CC=mpiCC"
 test.sh basictest.sh $src_dir -co: $co -mo: $mo -ro: $ro -eo: $eo
 renametest.sh basictest $output_dir/basictest-debug-cpp
 
-co="--with-insure --enable-debug --with-print-errors"
-MO="test"
-test.sh basictest.sh $src_dir -co: $co -mo: $MO -ro: $ro
-renametest.sh basictest $output_dir/basictest--with-insure1
-
-co="--with-insure --enable-debug --enable-global-partition"
-MO="test"
-test.sh basictest.sh $src_dir -co: $co -mo: $MO -ro: $ro
-renametest.sh basictest $output_dir/basictest--with-insure2
+# co="--with-insure --enable-debug --with-print-errors"
+# MO="test"
+# test.sh basictest.sh $src_dir -co: $co -mo: $MO -ro: $ro
+# renametest.sh basictest $output_dir/basictest--with-insure1
+# 
+# co="--with-insure --enable-debug --enable-global-partition"
+# MO="test"
+# test.sh basictest.sh $src_dir -co: $co -mo: $MO -ro: $ro
+# renametest.sh basictest $output_dir/basictest--with-insure2
+
+co="--enable-debug --with-print-errors"
+test.sh basictest.sh $src_dir -co: $co -mo: $mo -ro: $ro -rt -valgrind
+renametest.sh basictest $output_dir/basictest--valgrind1
+
+co="--enable-debug --enable-global-partition"
+test.sh basictest.sh $src_dir -co: $co -mo: $mo -ro: $ro -rt -valgrind
+renametest.sh basictest $output_dir/basictest--valgrind2
 
 co="--without-MPI"
 test.sh basictest.sh $src_dir -co: $co -mo: $mo
diff --git a/AUTOTEST/make.filters b/AUTOTEST/make.filters
index f1686be..478bba7 100644
--- a/AUTOTEST/make.filters
+++ b/AUTOTEST/make.filters
@@ -14,3 +14,4 @@ Additional optimization may be attained by recompiling and specifying MAXMEM opt
 WARNING in dReadValues: Infinite loop.  Program may not stop.
 WARNING in dReadVector: Infinite loop.  Program may not stop.
 icc: command line warning .*: overriding
+WARNING: no debugging flags detected
diff --git a/CHANGELOG b/CHANGELOG
index ccc6023..b659fda 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -15,6 +15,16 @@
 # recent release.
 #=============================================================================
 
+Version 2.11.2 released 2017/03/13
+
+- Changed the defaults in hypre to HMIS with ext+i(4) interpolation
+
+- Added a routine HYPRE_BoomerAMGSetOldDefault to easily get old defaults
+
+- Added Caliper instrumentation
+
+- Various bug fixes
+
 Version 2.11.1 released 2016/06/09
 
 - Fixed one more bug related to SStructSetSharedPart and SetNeighborPart
diff --git a/docs/HYPRE_ref_manual.pdf b/docs/HYPRE_ref_manual.pdf
index e851816..78de12f 100644
Binary files a/docs/HYPRE_ref_manual.pdf and b/docs/HYPRE_ref_manual.pdf differ
diff --git a/docs/HYPRE_usr_manual.pdf b/docs/HYPRE_usr_manual.pdf
index 77e18d6..a21b49e 100644
Binary files a/docs/HYPRE_usr_manual.pdf and b/docs/HYPRE_usr_manual.pdf differ
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index fb9b1ae..782dd52 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -2,8 +2,8 @@ cmake_minimum_required (VERSION 2.8.8)
 project (hypre)
 
 # The version number.
-set (HYPRE_VERSION 2.11.1)
-set (HYPRE_DATE    2016/06/09)
+set (HYPRE_VERSION 2.11.2)
+set (HYPRE_DATE    2017/03/13)
 set (HYPRE_TIME    00:00:00)
 set (HYPRE_BUGS    hypre-support at llnl.gov)
 set (HYPRE_SRCDIR  "${PROJECT_SOURCE_DIR}")
@@ -33,6 +33,7 @@ option(HYPRE_NO_GLOBAL_PARTITION  "Use assumed partition" ON)
 option(HYPRE_PRINT_ERRORS         "Print HYPRE errors" OFF)
 option(HYPRE_USING_OPENMP         "Use OpenMP" OFF)
 option(HYPRE_USING_FEI            "Use FEI" ON)
+option(HYPRE_USING_CALIPER        "Use Caliper" OFF)  # TODO: Finish this cmake feature
 
 if (HYPRE_SHARED)
   # FEI doesn't currently compile with shared
@@ -47,6 +48,7 @@ if (HYPRE_SEQUENTIAL)
   set (HYPRE_NO_GLOBAL_PARTITION OFF CACHE BOOL "" FORCE)
 endif ()
 
+#   --with-examples         Build example codes.
 #   --with-fei              Use internal FEI routines.
 #   --with-superlu          Use internal SuperLU routines.
 #   --with-mli              Use MLI
@@ -795,6 +797,8 @@ if (MSVC)
   set_source_files_properties (parcsr_ls/par_rap.c              PROPERTIES COMPILE_FLAGS /TP)
   set_source_files_properties (parcsr_ls/par_relax.c            PROPERTIES COMPILE_FLAGS /TP)
   set_source_files_properties (parcsr_ls/par_strength.c         PROPERTIES COMPILE_FLAGS /TP)
+  #Fix issue with visual studio 2013
+  set_source_files_properties (struct_ls/pfmg3_setup_rap.c      PROPERTIES COMPILE_FLAGS /Od)
 endif ()
 
 if (HYPRE_USING_FEI)
diff --git a/src/FEI_mv/SuperLU/SRC/dgsequ.c b/src/FEI_mv/SuperLU/SRC/dgsequ.c
index 515e8c4..a42fe5c 100644
--- a/src/FEI_mv/SuperLU/SRC/dgsequ.c
+++ b/src/FEI_mv/SuperLU/SRC/dgsequ.c
@@ -18,7 +18,9 @@
 
 #include <math.h>
 #include "slu_ddefs.h"
+#ifdef HYPRE_USING_HYPRE_LAPACK
 #include "hypre_lapack.h"
+#endif
 
 void
 dgsequ(SuperMatrix *A, double *r, double *c, double *rowcnd,
diff --git a/src/FEI_mv/SuperLU/SRC/dgssvx.c b/src/FEI_mv/SuperLU/SRC/dgssvx.c
index 1755e88..bf22b41 100644
--- a/src/FEI_mv/SuperLU/SRC/dgssvx.c
+++ b/src/FEI_mv/SuperLU/SRC/dgssvx.c
@@ -13,7 +13,9 @@
 
 
 #include "slu_ddefs.h"
+#ifdef HYPRE_USING_HYPRE_LAPACK
 #include "hypre_lapack.h"
+#endif
 
 void
 dgssvx(superlu_options_t *options, SuperMatrix *A, int *perm_c, int *perm_r,
diff --git a/src/FEI_mv/SuperLU/SRC/dlaqgs.c b/src/FEI_mv/SuperLU/SRC/dlaqgs.c
index 555332d..ccb7218 100644
--- a/src/FEI_mv/SuperLU/SRC/dlaqgs.c
+++ b/src/FEI_mv/SuperLU/SRC/dlaqgs.c
@@ -17,7 +17,9 @@
 
 #include <math.h>
 #include "slu_ddefs.h"
+#ifdef HYPRE_USING_HYPRE_LAPACK
 #include "hypre_lapack.h"
+#endif
 
 void
 dlaqgs(SuperMatrix *A, double *r, double *c, 
diff --git a/src/FEI_mv/SuperLU/SRC/dpivotgrowth.c b/src/FEI_mv/SuperLU/SRC/dpivotgrowth.c
index 49aaf8c..cd8f157 100644
--- a/src/FEI_mv/SuperLU/SRC/dpivotgrowth.c
+++ b/src/FEI_mv/SuperLU/SRC/dpivotgrowth.c
@@ -13,7 +13,9 @@
 
 #include <math.h>
 #include "slu_ddefs.h"
+#ifdef HYPRE_USING_HYPRE_LAPACK
 #include "hypre_lapack.h"
+#endif
 
 double
 dPivotGrowth(int ncols, SuperMatrix *A, int *perm_c, 
diff --git a/src/FEI_mv/fei-hypre/HYPRE_FEI_includes.h b/src/FEI_mv/fei-hypre/HYPRE_FEI_includes.h
old mode 100644
new mode 100755
index cdd9da4..7189268
--- a/src/FEI_mv/fei-hypre/HYPRE_FEI_includes.h
+++ b/src/FEI_mv/fei-hypre/HYPRE_FEI_includes.h
@@ -23,11 +23,18 @@
 
 #ifndef NOFEI
 
+#ifdef FEI2_1
+#include "fei_defs.h"
+#include "base/Data.h"
+#include "base/Lookup.h"
+#include "base/LinearSystemCore.h"
+#else
 //New FEI 2.23.02
 #include "fei_defs.h"
 #include "fei_Data.hpp"
 #include "fei_Lookup.hpp"
 #include "fei_LinearSystemCore.hpp"
+#endif
 
 #else
 
diff --git a/src/FEI_mv/fei-hypre/HYPRE_LinSysCore.cxx b/src/FEI_mv/fei-hypre/HYPRE_LinSysCore.cxx
index 568c50a..da48ac1 100644
--- a/src/FEI_mv/fei-hypre/HYPRE_LinSysCore.cxx
+++ b/src/FEI_mv/fei-hypre/HYPRE_LinSysCore.cxx
@@ -678,7 +678,8 @@ int HYPRE_LinSysCore::setLookup(Lookup& lookup)
    // set the lookup object and initialize the MLI_FEData object
    //-------------------------------------------------------------------
 
-   if (&lookup == NULL) return (0);
+   // RDF: The following line doesn't make sense and generates warnings with some compilers
+   //if (&lookup == NULL) return (0);
    lookup_ = &lookup;
    haveLookup_ = 1;
 
diff --git a/src/FEI_mv/femli/mli_utils.c b/src/FEI_mv/femli/mli_utils.c
index 89695a8..aa4a71c 100644
--- a/src/FEI_mv/femli/mli_utils.c
+++ b/src/FEI_mv/femli/mli_utils.c
@@ -27,7 +27,9 @@
 #include "mli_utils.h"
 #include "HYPRE_IJ_mv.h"
 #include "../fei-hypre/HYPRE_parcsr_fgmres.h"
+#ifdef HYPRE_USING_HYPRE_LAPACK
 #include "hypre_lapack.h"
+#endif
 
 /*--------------------------------------------------------------------------
  * external function 
diff --git a/src/IJ_mv/IJMatrix_parcsr.c b/src/IJ_mv/IJMatrix_parcsr.c
index 6c9b639..c97da09 100644
--- a/src/IJ_mv/IJMatrix_parcsr.c
+++ b/src/IJ_mv/IJMatrix_parcsr.c
@@ -2119,7 +2119,8 @@ hypre_IJMatrixAssembleOffProcValsParCSR( hypre_IJMatrix *matrix,
 
    hypre_TFree(num_elements_total);
 
-   void_contact_buf = hypre_MAlloc(storage*obj_size_bytes);
+   /*void_contact_buf = hypre_MAlloc(storage*obj_size_bytes);*/
+   void_contact_buf = hypre_CAlloc(storage, obj_size_bytes);
    index_ptr = void_contact_buf; /* step through with this index */
 
    /* for each proc: #rows, row #, no. elements, 
@@ -2832,11 +2833,11 @@ hypre_IJMatrixAssembleParCSR(hypre_IJMatrix *matrix)
          hypre_CSRMatrixNumCols(offd) = num_cols_offd;    
          hypre_TFree(aux_offd_j);
       }
-      hypre_AuxParCSRMatrixDestroy(aux_matrix);
-      hypre_IJMatrixTranslator(matrix) = NULL;
       hypre_IJMatrixAssembleFlag(matrix) = 1;
    }
 
+   hypre_AuxParCSRMatrixDestroy(aux_matrix);
+   hypre_IJMatrixTranslator(matrix) = NULL;
    return hypre_error_flag;
 }
 
diff --git a/src/Makefile b/src/Makefile
index 9a63e81..cba466f 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -140,7 +140,16 @@ help:
 	@echo "     "
 	@echo "check:"
 	@echo "     runs a small driver test to verify a working library"
-	@echo "     use CHECKRUN=<mpirun routine> if needed"
+	@echo "     use CHECKRUN=<mpirun routine> if needed combined with"
+	@echo "     PARMS='-P px py pz' where px*py*pz must be number of"
+	@echo "     processes set in CHECKRUN"
+	@echo "     "
+	@echo "     "
+	@echo "checkpar:"
+	@echo "     runs several regression test to verify a working library."
+	@echo "     Use parameter CHECKRUN='<mpirun routine -n>' if needed."
+	@echo "     If CHECKRUN is not set, 'mpirun -np' is used."
+	@echo "     "
 	@echo "     "
 	@echo "************************************************************"
 
@@ -153,11 +162,28 @@ check:
 	@ \
 	echo "Checking the library ..."; \
 	(cd test; $(MAKE) all); \
-	(cd test; $(CHECKRUN) ij 2> ij.err); \
-	(cd test; $(CHECKRUN) struct 2> struct.err); \
-	(cd test; cp -f TEST_sstruct/sstruct.in.default .; $(CHECKRUN) sstruct 2> sstruct.err); \
+	(cd test; $(CHECKRUN) ./ij $(PARMS) 2> ij.err); \
+	(cd test; $(CHECKRUN) ./struct $(PARMS) 2> struct.err); \
+	(cd test; cp -f TEST_sstruct/sstruct.in.default .; $(CHECKRUN) ./sstruct $(PARMS) 2> sstruct.err); \
 	(cd test; ls -l ij.err struct.err sstruct.err)
 
+checkpar:
+	@ \
+        echo "Checking the library ..."; \
+        (cd test; $(MAKE) all); \
+        echo "Testing IJ ..."; \
+        (cd test; ./runtest.sh -tol 1.e-06 -mpi "$(CHECKRUN)" TEST_ij/solvers.sh); \
+        (cd test; ./checktest.sh); \
+        (cd test; ./cleantest.sh); \
+        echo "Testing Struct ..."; \
+        (cd test; ./runtest.sh -tol 1.e-06 -mpi "$(CHECKRUN)" TEST_struct/solvers.sh); \
+        (cd test; ./checktest.sh); \
+        (cd test; ./cleantest.sh); \
+        echo "Testing SStruct ..."; \
+        (cd test; ./runtest.sh -tol 1.e-06 -mpi "$(CHECKRUN)" TEST_sstruct/solvers.sh); \
+        (cd test; ./checktest.sh); \
+        (cd test; ./cleantest.sh); 
+
 install: all
 	@ \
 	echo "Installing hypre ..."; \
diff --git a/src/config/HYPRE_config.h.cmake.in b/src/config/HYPRE_config.h.cmake.in
index 99da41c..3985e80 100644
--- a/src/config/HYPRE_config.h.cmake.in
+++ b/src/config/HYPRE_config.h.cmake.in
@@ -46,6 +46,9 @@
 /* Use OpenMP */
 #cmakedefine HYPRE_USING_OPENMP
 
+/* Use Caliper instrumentation */
+#cmakedefine HYPRE_USING_CALIPER
+
 /* #undef HYPRE_HAVE_MPI */
 /* #undef HYPRE_HAVE_MPI_COMM_F2C */
 
diff --git a/src/config/HYPRE_config.h.in b/src/config/HYPRE_config.h.in
index df80bb3..a7df6ef 100644
--- a/src/config/HYPRE_config.h.in
+++ b/src/config/HYPRE_config.h.in
@@ -114,3 +114,6 @@
 
 /* As HYPRE_FC_FUNC, but for C identifiers containing underscores. */
 #undef FC_FUNC_
+
+/* Define to 1 if Caliper instrumentation is enabled */
+#undef HYPRE_USING_CALIPER
diff --git a/src/config/Makefile.config.in b/src/config/Makefile.config.in
index be77e92..4ceb255 100644
--- a/src/config/Makefile.config.in
+++ b/src/config/Makefile.config.in
@@ -56,19 +56,19 @@ HYPRE_LIB_SUFFIX = @HYPRE_LIBSUFFIX@
 	$(CXX) $(CXXFLAGS) -c $<
 
 FC       = @FC@
-FFLAGS    = @FFLAGS@ @FCFLAGS@ $(FC_COMPILE_FLAGS)
+FFLAGS   = @FFLAGS@ @FCFLAGS@ $(FC_COMPILE_FLAGS)
 
-CC        = @CC@
-CFLAGS    = @CFLAGS@ @DEFS@ $(C_COMPILE_FLAGS)
+CC       = @CC@
+CFLAGS   = @CFLAGS@ @DEFS@ $(C_COMPILE_FLAGS)
 
-CXX       = @CXX@
-CXXFLAGS  = @CXXFLAGS@ @DEFS@ $(CXX_COMPILE_FLAGS)
+CXX      = @CXX@
+CXXFLAGS = @CXXFLAGS@ @DEFS@ $(CXX_COMPILE_FLAGS)
 
 LINK_FC  = @LINK_FC@
-LINK_CC   = @LINK_CC@
-LINK_CXX  = @LINK_CXX@
+LINK_CC  = @LINK_CC@
+LINK_CXX = @LINK_CXX@
 
-BUILD_FC_SHARED    = @BUILD_FC_SHARED@
+BUILD_FC_SHARED     = @BUILD_FC_SHARED@
 BUILD_CC_SHARED     = @BUILD_CC_SHARED@
 BUILD_CXX_SHARED    = @BUILD_CXX_SHARED@
 SHARED_COMPILE_FLAG = @SHARED_COMPILE_FLAG@
@@ -85,9 +85,11 @@ AR     = @AR@
 RANLIB = @RANLIB@
 
 LDFLAGS = @LDFLAGS@
-LIBS    = @LIBS@
+LIBS    = @LIBS@ @CALIPER_LIBS@
 FLIBS   = @FLIBS@
 
+INCLUDES = @CALIPER_INCLUDE@
+
 ##################################################################
 ##  LAPACK Library Flags 
 ##################################################################
@@ -113,6 +115,12 @@ MPILIBS    = @MPILIBS@
 MPIFLAGS   = @MPIFLAGS@
 
 ##################################################################
+##  Caliper options
+##################################################################
+CALIPER_INCLUDE = @CALIPER_INCLUDE@
+CALIPER_LIBS    = @CALIPER_LIBS@
+
+##################################################################
 ##  FEI options
 ##################################################################
 HYPRE_FEI_SRC_DIR       = @HYPRE_FEI_SRC_DIR@
diff --git a/src/config/config.guess b/src/config/config.guess
index ad5281e..2e9ad7f 100755
--- a/src/config/config.guess
+++ b/src/config/config.guess
@@ -1,13 +1,12 @@
 #! /bin/sh
 # Attempt to guess a canonical system name.
-#   Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
-#   2000, 2001, 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+#   Copyright 1992-2016 Free Software Foundation, Inc.
 
-timestamp='2005-08-03'
+timestamp='2016-10-02'
 
 # This file is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful, but
@@ -16,26 +15,22 @@ timestamp='2005-08-03'
 # General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA
-# 02110-1301, USA.
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
 #
 # As a special exception to the GNU General Public License, if you
 # distribute this file as part of a program that contains a
 # configuration script generated by Autoconf, you may include it under
-# the same distribution terms that you use for the rest of that program.
-
-
-# Originally written by Per Bothner <per at bothner.com>.
-# Please send patches to <config-patches at gnu.org>.  Submit a context
-# diff and a properly formatted ChangeLog entry.
+# the same distribution terms that you use for the rest of that
+# program.  This Exception is an additional permission under section 7
+# of the GNU General Public License, version 3 ("GPLv3").
 #
-# This script attempts to guess a canonical system name similar to
-# config.sub.  If it succeeds, it prints the system name on stdout, and
-# exits with 0.  Otherwise, it exits with 1.
+# Originally written by Per Bothner; maintained since 2000 by Ben Elliston.
 #
-# The plan is that this can be called by configure scripts if you
-# don't specify an explicit build system type.
+# You can get the latest version of this script from:
+# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess
+#
+# Please send patches to <config-patches at gnu.org>.
+
 
 me=`echo "$0" | sed -e 's,.*/,,'`
 
@@ -55,8 +50,7 @@ version="\
 GNU config.guess ($timestamp)
 
 Originally written by Per Bothner.
-Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005
-Free Software Foundation, Inc.
+Copyright 1992-2016 Free Software Foundation, Inc.
 
 This is free software; see the source for copying conditions.  There is NO
 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@@ -106,7 +100,7 @@ set_cc_for_build='
 trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) && exit \$exitcode" 0 ;
 trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15 ;
 : ${TMPDIR=/tmp} ;
- { tmp=`(umask 077 && mktemp -d -q "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } ||
+ { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } ||
  { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp) ; } ||
  { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir $tmp) && echo "Warning: creating insecure temp directory" >&2 ; } ||
  { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } ;
@@ -138,12 +132,33 @@ UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown
 UNAME_SYSTEM=`(uname -s) 2>/dev/null`  || UNAME_SYSTEM=unknown
 UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown
 
+case "${UNAME_SYSTEM}" in
+Linux|GNU|GNU/*)
+	# If the system lacks a compiler, then just pick glibc.
+	# We could probably try harder.
+	LIBC=gnu
+
+	eval $set_cc_for_build
+	cat <<-EOF > $dummy.c
+	#include <features.h>
+	#if defined(__UCLIBC__)
+	LIBC=uclibc
+	#elif defined(__dietlibc__)
+	LIBC=dietlibc
+	#else
+	LIBC=gnu
+	#endif
+	EOF
+	eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC' | sed 's, ,,g'`
+	;;
+esac
+
 # Note: order is significant - the case branches are not exclusive.
 
 case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
     *:NetBSD:*:*)
 	# NetBSD (nbsd) targets should (where applicable) match one or
-	# more of the tupples: *-*-netbsdelf*, *-*-netbsdaout*,
+	# more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*,
 	# *-*-netbsdecoff* and *-*-netbsd*.  For targets that recently
 	# switched to ELF, *-*-netbsd* would select the old
 	# object file format.  This provides both forward
@@ -153,22 +168,33 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 	# Note: NetBSD doesn't particularly care about the vendor
 	# portion of the name.  We always set it to "unknown".
 	sysctl="sysctl -n hw.machine_arch"
-	UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \
-	    /usr/sbin/$sysctl 2>/dev/null || echo unknown)`
+	UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \
+	    /sbin/$sysctl 2>/dev/null || \
+	    /usr/sbin/$sysctl 2>/dev/null || \
+	    echo unknown)`
 	case "${UNAME_MACHINE_ARCH}" in
 	    armeb) machine=armeb-unknown ;;
 	    arm*) machine=arm-unknown ;;
 	    sh3el) machine=shl-unknown ;;
 	    sh3eb) machine=sh-unknown ;;
+	    sh5el) machine=sh5le-unknown ;;
+	    earmv*)
+		arch=`echo ${UNAME_MACHINE_ARCH} | sed -e 's,^e\(armv[0-9]\).*$,\1,'`
+		endian=`echo ${UNAME_MACHINE_ARCH} | sed -ne 's,^.*\(eb\)$,\1,p'`
+		machine=${arch}${endian}-unknown
+		;;
 	    *) machine=${UNAME_MACHINE_ARCH}-unknown ;;
 	esac
 	# The Operating System including object format, if it has switched
-	# to ELF recently, or will in the future.
+	# to ELF recently (or will in the future) and ABI.
 	case "${UNAME_MACHINE_ARCH}" in
+	    earm*)
+		os=netbsdelf
+		;;
 	    arm*|i386|m68k|ns32k|sh3*|sparc|vax)
 		eval $set_cc_for_build
 		if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \
-			| grep __ELF__ >/dev/null
+			| grep -q __ELF__
 		then
 		    # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout).
 		    # Return netbsd for either.  FIX?
@@ -178,7 +204,14 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 		fi
 		;;
 	    *)
-	        os=netbsd
+		os=netbsd
+		;;
+	esac
+	# Determine ABI tags.
+	case "${UNAME_MACHINE_ARCH}" in
+	    earm*)
+		expr='s/^earmv[0-9]/-eabi/;s/eb$//'
+		abi=`echo ${UNAME_MACHINE_ARCH} | sed -e "$expr"`
 		;;
 	esac
 	# The OS release
@@ -191,34 +224,48 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 		release='-gnu'
 		;;
 	    *)
-		release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'`
+		release=`echo ${UNAME_RELEASE} | sed -e 's/[-_].*//' | cut -d. -f1,2`
 		;;
 	esac
 	# Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM:
 	# contains redundant information, the shorter form:
 	# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used.
-	echo "${machine}-${os}${release}"
+	echo "${machine}-${os}${release}${abi}"
+	exit ;;
+    *:Bitrig:*:*)
+	UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'`
+	echo ${UNAME_MACHINE_ARCH}-unknown-bitrig${UNAME_RELEASE}
 	exit ;;
     *:OpenBSD:*:*)
 	UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'`
 	echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE}
 	exit ;;
+    *:LibertyBSD:*:*)
+	UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'`
+	echo ${UNAME_MACHINE_ARCH}-unknown-libertybsd${UNAME_RELEASE}
+	exit ;;
     *:ekkoBSD:*:*)
 	echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE}
 	exit ;;
+    *:SolidBSD:*:*)
+	echo ${UNAME_MACHINE}-unknown-solidbsd${UNAME_RELEASE}
+	exit ;;
     macppc:MirBSD:*:*)
-	echo powerppc-unknown-mirbsd${UNAME_RELEASE}
+	echo powerpc-unknown-mirbsd${UNAME_RELEASE}
 	exit ;;
     *:MirBSD:*:*)
 	echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE}
 	exit ;;
+    *:Sortix:*:*)
+	echo ${UNAME_MACHINE}-unknown-sortix
+	exit ;;
     alpha:OSF1:*:*)
 	case $UNAME_RELEASE in
 	*4.0)
 		UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'`
 		;;
 	*5.*)
-	        UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'`
+		UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'`
 		;;
 	esac
 	# According to Compaq, /usr/sbin/psrinfo has been available on
@@ -228,43 +275,46 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 	ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^  The alpha \(.*\) processor.*$/\1/p' | head -n 1`
 	case "$ALPHA_CPU_TYPE" in
 	    "EV4 (21064)")
-		UNAME_MACHINE="alpha" ;;
+		UNAME_MACHINE=alpha ;;
 	    "EV4.5 (21064)")
-		UNAME_MACHINE="alpha" ;;
+		UNAME_MACHINE=alpha ;;
 	    "LCA4 (21066/21068)")
-		UNAME_MACHINE="alpha" ;;
+		UNAME_MACHINE=alpha ;;
 	    "EV5 (21164)")
-		UNAME_MACHINE="alphaev5" ;;
+		UNAME_MACHINE=alphaev5 ;;
 	    "EV5.6 (21164A)")
-		UNAME_MACHINE="alphaev56" ;;
+		UNAME_MACHINE=alphaev56 ;;
 	    "EV5.6 (21164PC)")
-		UNAME_MACHINE="alphapca56" ;;
+		UNAME_MACHINE=alphapca56 ;;
 	    "EV5.7 (21164PC)")
-		UNAME_MACHINE="alphapca57" ;;
+		UNAME_MACHINE=alphapca57 ;;
 	    "EV6 (21264)")
-		UNAME_MACHINE="alphaev6" ;;
+		UNAME_MACHINE=alphaev6 ;;
 	    "EV6.7 (21264A)")
-		UNAME_MACHINE="alphaev67" ;;
+		UNAME_MACHINE=alphaev67 ;;
 	    "EV6.8CB (21264C)")
-		UNAME_MACHINE="alphaev68" ;;
+		UNAME_MACHINE=alphaev68 ;;
 	    "EV6.8AL (21264B)")
-		UNAME_MACHINE="alphaev68" ;;
+		UNAME_MACHINE=alphaev68 ;;
 	    "EV6.8CX (21264D)")
-		UNAME_MACHINE="alphaev68" ;;
+		UNAME_MACHINE=alphaev68 ;;
 	    "EV6.9A (21264/EV69A)")
-		UNAME_MACHINE="alphaev69" ;;
+		UNAME_MACHINE=alphaev69 ;;
 	    "EV7 (21364)")
-		UNAME_MACHINE="alphaev7" ;;
+		UNAME_MACHINE=alphaev7 ;;
 	    "EV7.9 (21364A)")
-		UNAME_MACHINE="alphaev79" ;;
+		UNAME_MACHINE=alphaev79 ;;
 	esac
 	# A Pn.n version is a patched version.
 	# A Vn.n version is a released version.
 	# A Tn.n version is a released field test version.
 	# A Xn.n version is an unreleased experimental baselevel.
 	# 1.2 uses "1.2" for uname -r.
-	echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
-	exit ;;
+	echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`
+	# Reset EXIT trap before exiting to avoid spurious non-zero exit code.
+	exitcode=$?
+	trap '' 0
+	exit $exitcode ;;
     Alpha\ *:Windows_NT*:*)
 	# How do we know it's Interix rather than the generic POSIX subsystem?
 	# Should we change UNAME_MACHINE based on the output of uname instead
@@ -290,12 +340,12 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 	echo s390-ibm-zvmoe
 	exit ;;
     *:OS400:*:*)
-        echo powerpc-ibm-os400
+	echo powerpc-ibm-os400
 	exit ;;
     arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*)
 	echo arm-acorn-riscix${UNAME_RELEASE}
 	exit ;;
-    arm:riscos:*:*|arm:RISCOS:*:*)
+    arm*:riscos:*:*|arm*:RISCOS:*:*)
 	echo arm-unknown-riscos
 	exit ;;
     SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*)
@@ -319,14 +369,33 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 	case `/usr/bin/uname -p` in
 	    sparc) echo sparc-icl-nx7; exit ;;
 	esac ;;
+    s390x:SunOS:*:*)
+	echo ${UNAME_MACHINE}-ibm-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit ;;
     sun4H:SunOS:5.*:*)
 	echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
 	exit ;;
     sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*)
 	echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
 	exit ;;
-    i86pc:SunOS:5.*:*)
-	echo i386-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+    i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*)
+	echo i386-pc-auroraux${UNAME_RELEASE}
+	exit ;;
+    i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*)
+	eval $set_cc_for_build
+	SUN_ARCH=i386
+	# If there is a compiler, see if it is configured for 64-bit objects.
+	# Note that the Sun cc does not turn __LP64__ into 1 like gcc does.
+	# This test works for both compilers.
+	if [ "$CC_FOR_BUILD" != no_compiler_found ]; then
+	    if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \
+		(CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
+		grep IS_64BIT_ARCH >/dev/null
+	    then
+		SUN_ARCH=x86_64
+	    fi
+	fi
+	echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
 	exit ;;
     sun4*:SunOS:6*:*)
 	# According to config.sub, this is the proper way to canonicalize
@@ -348,7 +417,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 	exit ;;
     sun*:*:4.2BSD:*)
 	UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null`
-	test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3
+	test "x${UNAME_RELEASE}" = x && UNAME_RELEASE=3
 	case "`/bin/arch`" in
 	    sun3)
 		echo m68k-sun-sunos${UNAME_RELEASE}
@@ -370,23 +439,23 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
     # MiNT.  But MiNT is downward compatible to TOS, so this should
     # be no problem.
     atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*)
-        echo m68k-atari-mint${UNAME_RELEASE}
+	echo m68k-atari-mint${UNAME_RELEASE}
 	exit ;;
     atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*)
 	echo m68k-atari-mint${UNAME_RELEASE}
-        exit ;;
+	exit ;;
     *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*)
-        echo m68k-atari-mint${UNAME_RELEASE}
+	echo m68k-atari-mint${UNAME_RELEASE}
 	exit ;;
     milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*)
-        echo m68k-milan-mint${UNAME_RELEASE}
-        exit ;;
+	echo m68k-milan-mint${UNAME_RELEASE}
+	exit ;;
     hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*)
-        echo m68k-hades-mint${UNAME_RELEASE}
-        exit ;;
+	echo m68k-hades-mint${UNAME_RELEASE}
+	exit ;;
     *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*)
-        echo m68k-unknown-mint${UNAME_RELEASE}
-        exit ;;
+	echo m68k-unknown-mint${UNAME_RELEASE}
+	exit ;;
     m68k:machten:*:*)
 	echo m68k-apple-machten${UNAME_RELEASE}
 	exit ;;
@@ -456,8 +525,8 @@ EOF
 	echo m88k-motorola-sysv3
 	exit ;;
     AViiON:dgux:*:*)
-        # DG/UX returns AViiON for all architectures
-        UNAME_PROCESSOR=`/usr/bin/uname -p`
+	# DG/UX returns AViiON for all architectures
+	UNAME_PROCESSOR=`/usr/bin/uname -p`
 	if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ]
 	then
 	    if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \
@@ -470,7 +539,7 @@ EOF
 	else
 	    echo i586-dg-dgux${UNAME_RELEASE}
 	fi
- 	exit ;;
+	exit ;;
     M88*:DolphinOS:*:*)	# DolphinOS (SVR3)
 	echo m88k-dolphin-sysv3
 	exit ;;
@@ -527,15 +596,16 @@ EOF
 		echo rs6000-ibm-aix3.2
 	fi
 	exit ;;
-    *:AIX:*:[45])
+    *:AIX:*:[4567])
 	IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'`
 	if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then
 		IBM_ARCH=rs6000
 	else
 		IBM_ARCH=powerpc
 	fi
-	if [ -x /usr/bin/oslevel ] ; then
-		IBM_REV=`/usr/bin/oslevel`
+	if [ -x /usr/bin/lslpp ] ; then
+		IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc |
+			   awk -F: '{ print $3 }' | sed s/[0-9]*$/0/`
 	else
 		IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE}
 	fi
@@ -570,58 +640,58 @@ EOF
 	    9000/[678][0-9][0-9])
 		if [ -x /usr/bin/getconf ]; then
 		    sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null`
-                    sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null`
-                    case "${sc_cpu_version}" in
-                      523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0
-                      528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1
-                      532)                      # CPU_PA_RISC2_0
-                        case "${sc_kernel_bits}" in
-                          32) HP_ARCH="hppa2.0n" ;;
-                          64) HP_ARCH="hppa2.0w" ;;
-			  '') HP_ARCH="hppa2.0" ;;   # HP-UX 10.20
-                        esac ;;
-                    esac
+		    sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null`
+		    case "${sc_cpu_version}" in
+		      523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0
+		      528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1
+		      532)                      # CPU_PA_RISC2_0
+			case "${sc_kernel_bits}" in
+			  32) HP_ARCH=hppa2.0n ;;
+			  64) HP_ARCH=hppa2.0w ;;
+			  '') HP_ARCH=hppa2.0 ;;   # HP-UX 10.20
+			esac ;;
+		    esac
 		fi
 		if [ "${HP_ARCH}" = "" ]; then
 		    eval $set_cc_for_build
-		    sed 's/^              //' << EOF >$dummy.c
+		    sed 's/^		//' << EOF >$dummy.c
 
-              #define _HPUX_SOURCE
-              #include <stdlib.h>
-              #include <unistd.h>
+		#define _HPUX_SOURCE
+		#include <stdlib.h>
+		#include <unistd.h>
 
-              int main ()
-              {
-              #if defined(_SC_KERNEL_BITS)
-                  long bits = sysconf(_SC_KERNEL_BITS);
-              #endif
-                  long cpu  = sysconf (_SC_CPU_VERSION);
+		int main ()
+		{
+		#if defined(_SC_KERNEL_BITS)
+		    long bits = sysconf(_SC_KERNEL_BITS);
+		#endif
+		    long cpu  = sysconf (_SC_CPU_VERSION);
 
-                  switch (cpu)
-              	{
-              	case CPU_PA_RISC1_0: puts ("hppa1.0"); break;
-              	case CPU_PA_RISC1_1: puts ("hppa1.1"); break;
-              	case CPU_PA_RISC2_0:
-              #if defined(_SC_KERNEL_BITS)
-              	    switch (bits)
-              		{
-              		case 64: puts ("hppa2.0w"); break;
-              		case 32: puts ("hppa2.0n"); break;
-              		default: puts ("hppa2.0"); break;
-              		} break;
-              #else  /* !defined(_SC_KERNEL_BITS) */
-              	    puts ("hppa2.0"); break;
-              #endif
-              	default: puts ("hppa1.0"); break;
-              	}
-                  exit (0);
-              }
+		    switch (cpu)
+			{
+			case CPU_PA_RISC1_0: puts ("hppa1.0"); break;
+			case CPU_PA_RISC1_1: puts ("hppa1.1"); break;
+			case CPU_PA_RISC2_0:
+		#if defined(_SC_KERNEL_BITS)
+			    switch (bits)
+				{
+				case 64: puts ("hppa2.0w"); break;
+				case 32: puts ("hppa2.0n"); break;
+				default: puts ("hppa2.0"); break;
+				} break;
+		#else  /* !defined(_SC_KERNEL_BITS) */
+			    puts ("hppa2.0"); break;
+		#endif
+			default: puts ("hppa1.0"); break;
+			}
+		    exit (0);
+		}
 EOF
-		    (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy`
+		    (CCOPTS="" $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy`
 		    test -z "$HP_ARCH" && HP_ARCH=hppa
 		fi ;;
 	esac
-	if [ ${HP_ARCH} = "hppa2.0w" ]
+	if [ ${HP_ARCH} = hppa2.0w ]
 	then
 	    eval $set_cc_for_build
 
@@ -634,12 +704,12 @@ EOF
 	    # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess
 	    # => hppa64-hp-hpux11.23
 
-	    if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) |
-		grep __LP64__ >/dev/null
+	    if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) |
+		grep -q __LP64__
 	    then
-		HP_ARCH="hppa2.0w"
+		HP_ARCH=hppa2.0w
 	    else
-		HP_ARCH="hppa64"
+		HP_ARCH=hppa64
 	    fi
 	fi
 	echo ${HP_ARCH}-hp-hpux${HPUX_REV}
@@ -706,22 +776,22 @@ EOF
 	exit ;;
     C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*)
 	echo c1-convex-bsd
-        exit ;;
+	exit ;;
     C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*)
 	if getsysinfo -f scalar_acc
 	then echo c32-convex-bsd
 	else echo c2-convex-bsd
 	fi
-        exit ;;
+	exit ;;
     C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*)
 	echo c34-convex-bsd
-        exit ;;
+	exit ;;
     C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*)
 	echo c38-convex-bsd
-        exit ;;
+	exit ;;
     C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*)
 	echo c4-convex-bsd
-        exit ;;
+	exit ;;
     CRAY*Y-MP:*:*:*)
 	echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
 	exit ;;
@@ -744,15 +814,15 @@ EOF
 	echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
 	exit ;;
     F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*)
-	FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
-        FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
-        FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'`
-        echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
-        exit ;;
+	FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`
+	FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'`
+	FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'`
+	echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
+	exit ;;
     5000:UNIX_System_V:4.*:*)
-        FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
-        FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'`
-        echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
+	FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'`
+	FUJITSU_REL=`echo ${UNAME_RELEASE} | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'`
+	echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
 	exit ;;
     i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*)
 	echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE}
@@ -764,27 +834,51 @@ EOF
 	echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE}
 	exit ;;
     *:FreeBSD:*:*)
-	echo ${UNAME_MACHINE}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`
+	UNAME_PROCESSOR=`/usr/bin/uname -p`
+	case ${UNAME_PROCESSOR} in
+	    amd64)
+		echo x86_64-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;;
+	    *)
+		echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;;
+	esac
 	exit ;;
     i*:CYGWIN*:*)
 	echo ${UNAME_MACHINE}-pc-cygwin
 	exit ;;
-    i*:MINGW*:*)
+    *:MINGW64*:*)
+	echo ${UNAME_MACHINE}-pc-mingw64
+	exit ;;
+    *:MINGW*:*)
 	echo ${UNAME_MACHINE}-pc-mingw32
 	exit ;;
+    *:MSYS*:*)
+	echo ${UNAME_MACHINE}-pc-msys
+	exit ;;
     i*:windows32*:*)
-    	# uname -m includes "-pc" on this system.
-    	echo ${UNAME_MACHINE}-mingw32
+	# uname -m includes "-pc" on this system.
+	echo ${UNAME_MACHINE}-mingw32
 	exit ;;
     i*:PW*:*)
 	echo ${UNAME_MACHINE}-pc-pw32
 	exit ;;
-    x86:Interix*:[34]*)
-	echo i586-pc-interix${UNAME_RELEASE}|sed -e 's/\..*//'
-	exit ;;
+    *:Interix*:*)
+	case ${UNAME_MACHINE} in
+	    x86)
+		echo i586-pc-interix${UNAME_RELEASE}
+		exit ;;
+	    authenticamd | genuineintel | EM64T)
+		echo x86_64-unknown-interix${UNAME_RELEASE}
+		exit ;;
+	    IA64)
+		echo ia64-unknown-interix${UNAME_RELEASE}
+		exit ;;
+	esac ;;
     [345]86:Windows_95:* | [345]86:Windows_98:* | [345]86:Windows_NT:*)
 	echo i${UNAME_MACHINE}-pc-mks
 	exit ;;
+    8664:Windows_NT:*)
+	echo x86_64-pc-mks
+	exit ;;
     i*:Windows_NT*:* | Pentium*:Windows_NT*:*)
 	# How do we know it's Interix rather than the generic POSIX subsystem?
 	# It also conflicts with pre-2.0 versions of AT&T UWIN. Should we
@@ -805,182 +899,169 @@ EOF
 	exit ;;
     *:GNU:*:*)
 	# the GNU system
-	echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'`
+	echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-${LIBC}`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'`
 	exit ;;
     *:GNU/*:*:*)
 	# other systems with GNU libc and userland
-	echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-gnu
+	echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC}
 	exit ;;
     i*86:Minix:*:*)
 	echo ${UNAME_MACHINE}-pc-minix
 	exit ;;
+    aarch64:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+	exit ;;
+    aarch64_be:Linux:*:*)
+	UNAME_MACHINE=aarch64_be
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+	exit ;;
+    alpha:Linux:*:*)
+	case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in
+	  EV5)   UNAME_MACHINE=alphaev5 ;;
+	  EV56)  UNAME_MACHINE=alphaev56 ;;
+	  PCA56) UNAME_MACHINE=alphapca56 ;;
+	  PCA57) UNAME_MACHINE=alphapca56 ;;
+	  EV6)   UNAME_MACHINE=alphaev6 ;;
+	  EV67)  UNAME_MACHINE=alphaev67 ;;
+	  EV68*) UNAME_MACHINE=alphaev68 ;;
+	esac
+	objdump --private-headers /bin/sh | grep -q ld.so.1
+	if test "$?" = 0 ; then LIBC=gnulibc1 ; fi
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+	exit ;;
+    arc:Linux:*:* | arceb:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+	exit ;;
     arm*:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	eval $set_cc_for_build
+	if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \
+	    | grep -q __ARM_EABI__
+	then
+	    echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+	else
+	    if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \
+		| grep -q __ARM_PCS_VFP
+	    then
+		echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabi
+	    else
+		echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabihf
+	    fi
+	fi
+	exit ;;
+    avr32*:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     cris:Linux:*:*)
-	echo cris-axis-linux-gnu
+	echo ${UNAME_MACHINE}-axis-linux-${LIBC}
 	exit ;;
     crisv32:Linux:*:*)
-	echo crisv32-axis-linux-gnu
+	echo ${UNAME_MACHINE}-axis-linux-${LIBC}
+	exit ;;
+    e2k:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     frv:Linux:*:*)
-    	echo frv-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+	exit ;;
+    hexagon:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+	exit ;;
+    i*86:Linux:*:*)
+	echo ${UNAME_MACHINE}-pc-linux-${LIBC}
 	exit ;;
     ia64:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+	exit ;;
+    k1om:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     m32r*:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     m68*:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
-    mips:Linux:*:*)
+    mips:Linux:*:* | mips64:Linux:*:*)
 	eval $set_cc_for_build
 	sed 's/^	//' << EOF >$dummy.c
 	#undef CPU
-	#undef mips
-	#undef mipsel
+	#undef ${UNAME_MACHINE}
+	#undef ${UNAME_MACHINE}el
 	#if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL)
-	CPU=mipsel
+	CPU=${UNAME_MACHINE}el
 	#else
 	#if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB)
-	CPU=mips
+	CPU=${UNAME_MACHINE}
 	#else
 	CPU=
 	#endif
 	#endif
 EOF
-	eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^CPU=`
-	test x"${CPU}" != x && { echo "${CPU}-unknown-linux-gnu"; exit; }
+	eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'`
+	test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; }
 	;;
-    mips64:Linux:*:*)
-	eval $set_cc_for_build
-	sed 's/^	//' << EOF >$dummy.c
-	#undef CPU
-	#undef mips64
-	#undef mips64el
-	#if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL)
-	CPU=mips64el
-	#else
-	#if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB)
-	CPU=mips64
-	#else
-	CPU=
-	#endif
-	#endif
-EOF
-	eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^CPU=`
-	test x"${CPU}" != x && { echo "${CPU}-unknown-linux-gnu"; exit; }
-	;;
-    or32:Linux:*:*)
-	echo or32-unknown-linux-gnu
+    mips64el:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
-    ppc:Linux:*:*)
-	echo powerpc-unknown-linux-gnu
+    openrisc*:Linux:*:*)
+	echo or1k-unknown-linux-${LIBC}
 	exit ;;
-    ppc64:Linux:*:*)
-	echo powerpc64-unknown-linux-gnu
+    or32:Linux:*:* | or1k*:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
-    alpha:Linux:*:*)
-	case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in
-	  EV5)   UNAME_MACHINE=alphaev5 ;;
-	  EV56)  UNAME_MACHINE=alphaev56 ;;
-	  PCA56) UNAME_MACHINE=alphapca56 ;;
-	  PCA57) UNAME_MACHINE=alphapca56 ;;
-	  EV6)   UNAME_MACHINE=alphaev6 ;;
-	  EV67)  UNAME_MACHINE=alphaev67 ;;
-	  EV68*) UNAME_MACHINE=alphaev68 ;;
-        esac
-	objdump --private-headers /bin/sh | grep ld.so.1 >/dev/null
-	if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi
-	echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC}
+    padre:Linux:*:*)
+	echo sparc-unknown-linux-${LIBC}
+	exit ;;
+    parisc64:Linux:*:* | hppa64:Linux:*:*)
+	echo hppa64-unknown-linux-${LIBC}
 	exit ;;
     parisc:Linux:*:* | hppa:Linux:*:*)
 	# Look for CPU level
 	case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in
-	  PA7*) echo hppa1.1-unknown-linux-gnu ;;
-	  PA8*) echo hppa2.0-unknown-linux-gnu ;;
-	  *)    echo hppa-unknown-linux-gnu ;;
+	  PA7*) echo hppa1.1-unknown-linux-${LIBC} ;;
+	  PA8*) echo hppa2.0-unknown-linux-${LIBC} ;;
+	  *)    echo hppa-unknown-linux-${LIBC} ;;
 	esac
 	exit ;;
-    parisc64:Linux:*:* | hppa64:Linux:*:*)
-	echo hppa64-unknown-linux-gnu
+    ppc64:Linux:*:*)
+	echo powerpc64-unknown-linux-${LIBC}
+	exit ;;
+    ppc:Linux:*:*)
+	echo powerpc-unknown-linux-${LIBC}
+	exit ;;
+    ppc64le:Linux:*:*)
+	echo powerpc64le-unknown-linux-${LIBC}
+	exit ;;
+    ppcle:Linux:*:*)
+	echo powerpcle-unknown-linux-${LIBC}
+	exit ;;
+    riscv32:Linux:*:* | riscv64:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     s390:Linux:*:* | s390x:Linux:*:*)
-	echo ${UNAME_MACHINE}-ibm-linux
+	echo ${UNAME_MACHINE}-ibm-linux-${LIBC}
 	exit ;;
     sh64*:Linux:*:*)
-    	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     sh*:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     sparc:Linux:*:* | sparc64:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+	exit ;;
+    tile*:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+	exit ;;
+    vax:Linux:*:*)
+	echo ${UNAME_MACHINE}-dec-linux-${LIBC}
 	exit ;;
     x86_64:Linux:*:*)
-	echo x86_64-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-pc-linux-${LIBC}
+	exit ;;
+    xtensa*:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
-    i*86:Linux:*:*)
-	# The BFD linker knows what the default object file format is, so
-	# first see if it will tell us. cd to the root directory to prevent
-	# problems with other programs or directories called `ld' in the path.
-	# Set LC_ALL=C to ensure ld outputs messages in English.
-	ld_supported_targets=`cd /; LC_ALL=C ld --help 2>&1 \
-			 | sed -ne '/supported targets:/!d
-				    s/[ 	][ 	]*/ /g
-				    s/.*supported targets: *//
-				    s/ .*//
-				    p'`
-        case "$ld_supported_targets" in
-	  elf32-i386)
-		TENTATIVE="${UNAME_MACHINE}-pc-linux-gnu"
-		;;
-	  a.out-i386-linux)
-		echo "${UNAME_MACHINE}-pc-linux-gnuaout"
-		exit ;;
-	  coff-i386)
-		echo "${UNAME_MACHINE}-pc-linux-gnucoff"
-		exit ;;
-	  "")
-		# Either a pre-BFD a.out linker (linux-gnuoldld) or
-		# one that does not give us useful --help.
-		echo "${UNAME_MACHINE}-pc-linux-gnuoldld"
-		exit ;;
-	esac
-	# Determine whether the default compiler is a.out or elf
-	eval $set_cc_for_build
-	sed 's/^	//' << EOF >$dummy.c
-	#include <features.h>
-	#ifdef __ELF__
-	# ifdef __GLIBC__
-	#  if __GLIBC__ >= 2
-	LIBC=gnu
-	#  else
-	LIBC=gnulibc1
-	#  endif
-	# else
-	LIBC=gnulibc1
-	# endif
-	#else
-	#ifdef __INTEL_COMPILER
-	LIBC=gnu
-	#else
-	LIBC=gnuaout
-	#endif
-	#endif
-	#ifdef __dietlibc__
-	LIBC=dietlibc
-	#endif
-EOF
-	eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^LIBC=`
-	test x"${LIBC}" != x && {
-		echo "${UNAME_MACHINE}-pc-linux-${LIBC}"
-		exit
-	}
-	test x"${TENTATIVE}" != x && { echo "${TENTATIVE}"; exit; }
-	;;
     i*86:DYNIX/ptx:4*:*)
 	# ptx 4.0 does uname -s correctly, with DYNIX/ptx in there.
 	# earlier versions are messed up and put the nodename in both
@@ -988,11 +1069,11 @@ EOF
 	echo i386-sequent-sysv4
 	exit ;;
     i*86:UNIX_SV:4.2MP:2.*)
-        # Unixware is an offshoot of SVR4, but it has its own version
-        # number series starting with 2...
-        # I am not positive that other SVR4 systems won't match this,
+	# Unixware is an offshoot of SVR4, but it has its own version
+	# number series starting with 2...
+	# I am not positive that other SVR4 systems won't match this,
 	# I just have to hope.  -- rms.
-        # Use sysv4.2uw... so that sysv4* matches it.
+	# Use sysv4.2uw... so that sysv4* matches it.
 	echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION}
 	exit ;;
     i*86:OS/2:*:*)
@@ -1009,7 +1090,7 @@ EOF
     i*86:syllable:*:*)
 	echo ${UNAME_MACHINE}-pc-syllable
 	exit ;;
-    i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.0*:*)
+    i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*)
 	echo i386-unknown-lynxos${UNAME_RELEASE}
 	exit ;;
     i*86:*DOS:*:*)
@@ -1024,7 +1105,7 @@ EOF
 	fi
 	exit ;;
     i*86:*:5:[678]*)
-    	# UnixWare 7.x, OpenUNIX and OpenServer 6.
+	# UnixWare 7.x, OpenUNIX and OpenServer 6.
 	case `/bin/uname -X | grep "^Machine"` in
 	    *486*)	     UNAME_MACHINE=i486 ;;
 	    *Pentium)	     UNAME_MACHINE=i586 ;;
@@ -1052,10 +1133,13 @@ EOF
 	exit ;;
     pc:*:*:*)
 	# Left here for compatibility:
-        # uname -m prints for DJGPP always 'pc', but it prints nothing about
-        # the processor, so we play safe by assuming i386.
-	echo i386-pc-msdosdjgpp
-        exit ;;
+	# uname -m prints for DJGPP always 'pc', but it prints nothing about
+	# the processor, so we play safe by assuming i586.
+	# Note: whatever this is, it MUST be the same as what config.sub
+	# prints for the "djgpp" host, or else GDB configure will decide that
+	# this is a cross-build.
+	echo i586-pc-msdosdjgpp
+	exit ;;
     Intel:Mach:3*:*)
 	echo i386-pc-mach3
 	exit ;;
@@ -1090,8 +1174,18 @@ EOF
 	/bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \
 	  && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;;
     3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*)
-        /bin/uname -p 2>/dev/null | grep 86 >/dev/null \
-          && { echo i486-ncr-sysv4; exit; } ;;
+	/bin/uname -p 2>/dev/null | grep 86 >/dev/null \
+	  && { echo i486-ncr-sysv4; exit; } ;;
+    NCR*:*:4.2:* | MPRAS*:*:4.2:*)
+	OS_REL='.3'
+	test -r /etc/.relid \
+	    && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid`
+	/bin/uname -p 2>/dev/null | grep 86 >/dev/null \
+	    && { echo i486-ncr-sysv4.3${OS_REL}; exit; }
+	/bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \
+	    && { echo i586-ncr-sysv4.3${OS_REL}; exit; }
+	/bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \
+	    && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;;
     m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*)
 	echo m68k-unknown-lynxos${UNAME_RELEASE}
 	exit ;;
@@ -1104,7 +1198,7 @@ EOF
     rs6000:LynxOS:2.*:*)
 	echo rs6000-unknown-lynxos${UNAME_RELEASE}
 	exit ;;
-    PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.0*:*)
+    PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*)
 	echo powerpc-unknown-lynxos${UNAME_RELEASE}
 	exit ;;
     SM[BE]S:UNIX_SV:*:*)
@@ -1124,10 +1218,10 @@ EOF
 		echo ns32k-sni-sysv
 	fi
 	exit ;;
-    PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort
-                      # says <Richard.M.Bartel at ccMail.Census.GOV>
-        echo i586-unisys-sysv4
-        exit ;;
+    PENTIUM:*:4.0*:*)	# Unisys `ClearPath HMP IX 4000' SVR4/MP effort
+			# says <Richard.M.Bartel at ccMail.Census.GOV>
+	echo i586-unisys-sysv4
+	exit ;;
     *:UNIX_System_V:4*:FTX*)
 	# From Gerald Hewes <hewes at openmarket.com>.
 	# How about differentiating between stratus architectures? -djm
@@ -1153,11 +1247,11 @@ EOF
 	exit ;;
     R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*)
 	if [ -d /usr/nec ]; then
-	        echo mips-nec-sysv${UNAME_RELEASE}
+		echo mips-nec-sysv${UNAME_RELEASE}
 	else
-	        echo mips-unknown-sysv${UNAME_RELEASE}
+		echo mips-unknown-sysv${UNAME_RELEASE}
 	fi
-        exit ;;
+	exit ;;
     BeBox:BeOS:*:*)	# BeOS running on hardware made by Be, PPC only.
 	echo powerpc-be-beos
 	exit ;;
@@ -1167,6 +1261,12 @@ EOF
     BePC:BeOS:*:*)	# BeOS running on Intel PC compatible.
 	echo i586-pc-beos
 	exit ;;
+    BePC:Haiku:*:*)	# Haiku running on Intel PC compatible.
+	echo i586-pc-haiku
+	exit ;;
+    x86_64:Haiku:*:*)
+	echo x86_64-unknown-haiku
+	exit ;;
     SX-4:SUPER-UX:*:*)
 	echo sx4-nec-superux${UNAME_RELEASE}
 	exit ;;
@@ -1176,6 +1276,18 @@ EOF
     SX-6:SUPER-UX:*:*)
 	echo sx6-nec-superux${UNAME_RELEASE}
 	exit ;;
+    SX-7:SUPER-UX:*:*)
+	echo sx7-nec-superux${UNAME_RELEASE}
+	exit ;;
+    SX-8:SUPER-UX:*:*)
+	echo sx8-nec-superux${UNAME_RELEASE}
+	exit ;;
+    SX-8R:SUPER-UX:*:*)
+	echo sx8r-nec-superux${UNAME_RELEASE}
+	exit ;;
+    SX-ACE:SUPER-UX:*:*)
+	echo sxace-nec-superux${UNAME_RELEASE}
+	exit ;;
     Power*:Rhapsody:*:*)
 	echo powerpc-apple-rhapsody${UNAME_RELEASE}
 	exit ;;
@@ -1184,15 +1296,36 @@ EOF
 	exit ;;
     *:Darwin:*:*)
 	UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown
-	case $UNAME_PROCESSOR in
-	    *86) UNAME_PROCESSOR=i686 ;;
-	    unknown) UNAME_PROCESSOR=powerpc ;;
-	esac
+	eval $set_cc_for_build
+	if test "$UNAME_PROCESSOR" = unknown ; then
+	    UNAME_PROCESSOR=powerpc
+	fi
+	if test `echo "$UNAME_RELEASE" | sed -e 's/\..*//'` -le 10 ; then
+	    if [ "$CC_FOR_BUILD" != no_compiler_found ]; then
+		if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \
+		    (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
+		    grep IS_64BIT_ARCH >/dev/null
+		then
+		    case $UNAME_PROCESSOR in
+			i386) UNAME_PROCESSOR=x86_64 ;;
+			powerpc) UNAME_PROCESSOR=powerpc64 ;;
+		    esac
+		fi
+	    fi
+	elif test "$UNAME_PROCESSOR" = i386 ; then
+	    # Avoid executing cc on OS X 10.9, as it ships with a stub
+	    # that puts up a graphical alert prompting to install
+	    # developer tools.  Any system running Mac OS X 10.7 or
+	    # later (Darwin 11 and later) is required to have a 64-bit
+	    # processor. This is not true of the ARM version of Darwin
+	    # that Apple uses in portable devices.
+	    UNAME_PROCESSOR=x86_64
+	fi
 	echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE}
 	exit ;;
     *:procnto*:*:* | *:QNX:[0123456789]*:*)
 	UNAME_PROCESSOR=`uname -p`
-	if test "$UNAME_PROCESSOR" = "x86"; then
+	if test "$UNAME_PROCESSOR" = x86; then
 		UNAME_PROCESSOR=i386
 		UNAME_MACHINE=pc
 	fi
@@ -1201,7 +1334,10 @@ EOF
     *:QNX:*:4*)
 	echo i386-pc-qnx
 	exit ;;
-    NSE-?:NONSTOP_KERNEL:*:*)
+    NEO-?:NONSTOP_KERNEL:*:*)
+	echo neo-tandem-nsk${UNAME_RELEASE}
+	exit ;;
+    NSE-*:NONSTOP_KERNEL:*:*)
 	echo nse-tandem-nsk${UNAME_RELEASE}
 	exit ;;
     NSR-?:NONSTOP_KERNEL:*:*)
@@ -1220,7 +1356,7 @@ EOF
 	# "uname -m" is not consistent, so use $cputype instead. 386
 	# is converted to i386 for consistency with other x86
 	# operating systems.
-	if test "$cputype" = "386"; then
+	if test "$cputype" = 386; then
 	    UNAME_MACHINE=i386
 	else
 	    UNAME_MACHINE="$cputype"
@@ -1246,13 +1382,13 @@ EOF
 	echo pdp10-unknown-its
 	exit ;;
     SEI:*:*:SEIUX)
-        echo mips-sei-seiux${UNAME_RELEASE}
+	echo mips-sei-seiux${UNAME_RELEASE}
 	exit ;;
     *:DragonFly:*:*)
 	echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`
 	exit ;;
     *:*VMS:*:*)
-    	UNAME_MACHINE=`(uname -p) 2>/dev/null`
+	UNAME_MACHINE=`(uname -p) 2>/dev/null`
 	case "${UNAME_MACHINE}" in
 	    A*) echo alpha-dec-vms ; exit ;;
 	    I*) echo ia64-dec-vms ; exit ;;
@@ -1262,176 +1398,36 @@ EOF
 	echo i386-pc-xenix
 	exit ;;
     i*86:skyos:*:*)
-	echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//'
+	echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE} | sed -e 's/ .*$//'`
 	exit ;;
-esac
-
-#echo '(No uname command or uname output not recognized.)' 1>&2
-#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2
-
-eval $set_cc_for_build
-cat >$dummy.c <<EOF
-#ifdef _SEQUENT_
-# include <sys/types.h>
-# include <sys/utsname.h>
-#endif
-main ()
-{
-#if defined (sony)
-#if defined (MIPSEB)
-  /* BFD wants "bsd" instead of "newsos".  Perhaps BFD should be changed,
-     I don't know....  */
-  printf ("mips-sony-bsd\n"); exit (0);
-#else
-#include <sys/param.h>
-  printf ("m68k-sony-newsos%s\n",
-#ifdef NEWSOS4
-          "4"
-#else
-	  ""
-#endif
-         ); exit (0);
-#endif
-#endif
-
-#if defined (__arm) && defined (__acorn) && defined (__unix)
-  printf ("arm-acorn-riscix\n"); exit (0);
-#endif
-
-#if defined (hp300) && !defined (hpux)
-  printf ("m68k-hp-bsd\n"); exit (0);
-#endif
-
-#if defined (NeXT)
-#if !defined (__ARCHITECTURE__)
-#define __ARCHITECTURE__ "m68k"
-#endif
-  int version;
-  version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`;
-  if (version < 4)
-    printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version);
-  else
-    printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version);
-  exit (0);
-#endif
-
-#if defined (MULTIMAX) || defined (n16)
-#if defined (UMAXV)
-  printf ("ns32k-encore-sysv\n"); exit (0);
-#else
-#if defined (CMU)
-  printf ("ns32k-encore-mach\n"); exit (0);
-#else
-  printf ("ns32k-encore-bsd\n"); exit (0);
-#endif
-#endif
-#endif
-
-#if defined (__386BSD__)
-  printf ("i386-pc-bsd\n"); exit (0);
-#endif
-
-#if defined (sequent)
-#if defined (i386)
-  printf ("i386-sequent-dynix\n"); exit (0);
-#endif
-#if defined (ns32000)
-  printf ("ns32k-sequent-dynix\n"); exit (0);
-#endif
-#endif
-
-#if defined (_SEQUENT_)
-    struct utsname un;
-
-    uname(&un);
-
-    if (strncmp(un.version, "V2", 2) == 0) {
-	printf ("i386-sequent-ptx2\n"); exit (0);
-    }
-    if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */
-	printf ("i386-sequent-ptx1\n"); exit (0);
-    }
-    printf ("i386-sequent-ptx\n"); exit (0);
-
-#endif
-
-#if defined (vax)
-# if !defined (ultrix)
-#  include <sys/param.h>
-#  if defined (BSD)
-#   if BSD == 43
-      printf ("vax-dec-bsd4.3\n"); exit (0);
-#   else
-#    if BSD == 199006
-      printf ("vax-dec-bsd4.3reno\n"); exit (0);
-#    else
-      printf ("vax-dec-bsd\n"); exit (0);
-#    endif
-#   endif
-#  else
-    printf ("vax-dec-bsd\n"); exit (0);
-#  endif
-# else
-    printf ("vax-dec-ultrix\n"); exit (0);
-# endif
-#endif
-
-#if defined (alliant) && defined (i860)
-  printf ("i860-alliant-bsd\n"); exit (0);
-#endif
-
-  exit (1);
-}
-EOF
-
-$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && SYSTEM_NAME=`$dummy` &&
-	{ echo "$SYSTEM_NAME"; exit; }
-
-# Apollos put the system type in the environment.
-
-test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit; }
-
-# Convex versions that predate uname can use getsysinfo(1)
-
-if [ -x /usr/convex/getsysinfo ]
-then
-    case `getsysinfo -f cpu_type` in
-    c1*)
-	echo c1-convex-bsd
+    i*86:rdos:*:*)
+	echo ${UNAME_MACHINE}-pc-rdos
 	exit ;;
-    c2*)
-	if getsysinfo -f scalar_acc
-	then echo c32-convex-bsd
-	else echo c2-convex-bsd
-	fi
+    i*86:AROS:*:*)
+	echo ${UNAME_MACHINE}-pc-aros
 	exit ;;
-    c34*)
-	echo c34-convex-bsd
-	exit ;;
-    c38*)
-	echo c38-convex-bsd
+    x86_64:VMkernel:*:*)
+	echo ${UNAME_MACHINE}-unknown-esx
 	exit ;;
-    c4*)
-	echo c4-convex-bsd
+    amd64:Isilon\ OneFS:*:*)
+	echo x86_64-unknown-onefs
 	exit ;;
-    esac
-fi
+esac
 
 cat >&2 <<EOF
 $0: unable to guess system type
 
-This script, last modified $timestamp, has failed to recognize
-the operating system you are using. It is advised that you
-download the most up to date version of the config scripts from
+This script (version $timestamp), has failed to recognize the
+operating system you are using. If your script is old, overwrite
+config.guess and config.sub with the latest versions from:
 
-  http://savannah.gnu.org/cgi-bin/viewcvs/*checkout*/config/config/config.guess
+  http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess
 and
-  http://savannah.gnu.org/cgi-bin/viewcvs/*checkout*/config/config/config.sub
+  http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub
 
-If the version you run ($0) is already up to date, please
-send the following data and any information you think might be
-pertinent to <config-patches at gnu.org> in order to provide the needed
-information to handle your system.
+If $0 has already been updated, send the following data and any
+information you think might be pertinent to config-patches at gnu.org to
+provide the necessary information to handle your system.
 
 config.guess timestamp = $timestamp
 
diff --git a/src/config/config.sub b/src/config/config.sub
index 1c366df..3478c1f 100755
--- a/src/config/config.sub
+++ b/src/config/config.sub
@@ -1,43 +1,40 @@
 #! /bin/sh
 # Configuration validation subroutine script.
-#   Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
-#   2000, 2001, 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+#   Copyright 1992-2016 Free Software Foundation, Inc.
 
-timestamp='2005-07-08'
+timestamp='2016-11-19'
 
-# This file is (in principle) common to ALL GNU software.
-# The presence of a machine in this file suggests that SOME GNU software
-# can handle that machine.  It does not imply ALL GNU software can.
-#
-# This file is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA
-# 02110-1301, USA.
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
 #
 # As a special exception to the GNU General Public License, if you
 # distribute this file as part of a program that contains a
 # configuration script generated by Autoconf, you may include it under
-# the same distribution terms that you use for the rest of that program.
+# the same distribution terms that you use for the rest of that
+# program.  This Exception is an additional permission under section 7
+# of the GNU General Public License, version 3 ("GPLv3").
 
 
-# Please send patches to <config-patches at gnu.org>.  Submit a context
-# diff and a properly formatted ChangeLog entry.
+# Please send patches to <config-patches at gnu.org>.
 #
 # Configuration subroutine to validate and canonicalize a configuration type.
 # Supply the specified configuration type as an argument.
 # If it is invalid, we print an error message on stderr and exit with code 1.
 # Otherwise, we print the canonical config type on stdout and succeed.
 
+# You can get the latest version of this script from:
+# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub
+
 # This file is supposed to be the same for all GNU packages
 # and recognize all the CPU types, system types and aliases
 # that are meaningful with *any* GNU software.
@@ -56,8 +53,7 @@ timestamp='2005-07-08'
 me=`echo "$0" | sed -e 's,.*/,,'`
 
 usage="\
-Usage: $0 [OPTION] CPU-MFR-OPSYS
-       $0 [OPTION] ALIAS
+Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS
 
 Canonicalize a configuration name.
 
@@ -71,8 +67,7 @@ Report bugs and patches to <config-patches at gnu.org>."
 version="\
 GNU config.sub ($timestamp)
 
-Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005
-Free Software Foundation, Inc.
+Copyright 1992-2016 Free Software Foundation, Inc.
 
 This is free software; see the source for copying conditions.  There is NO
 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@@ -119,11 +114,18 @@ esac
 # Here we must recognize all the valid KERNEL-OS combinations.
 maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'`
 case $maybe_os in
-  nto-qnx* | linux-gnu* | linux-dietlibc | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | \
-  kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* | storm-chaos* | os2-emx* | rtmk-nova*)
+  nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \
+  linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \
+  knetbsd*-gnu* | netbsd*-gnu* | netbsd*-eabi* | \
+  kopensolaris*-gnu* | cloudabi*-eabi* | \
+  storm-chaos* | os2-emx* | rtmk-nova*)
     os=-$maybe_os
     basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`
     ;;
+  android-linux)
+    os=-linux-android
+    basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`-unknown
+    ;;
   *)
     basic_machine=`echo $1 | sed 's/-[^-]*$//'`
     if [ $basic_machine != $1 ]
@@ -146,10 +148,13 @@ case $os in
 	-convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\
 	-c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \
 	-harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \
-	-apple | -axis | -knuth | -cray)
+	-apple | -axis | -knuth | -cray | -microblaze*)
 		os=
 		basic_machine=$1
 		;;
+	-bluegene*)
+		os=-cnk
+		;;
 	-sim | -cisco | -oki | -wec | -winbond)
 		os=
 		basic_machine=$1
@@ -164,13 +169,17 @@ case $os in
 		os=-chorusos
 		basic_machine=$1
 		;;
- 	-chorusrdb)
- 		os=-chorusrdb
+	-chorusrdb)
+		os=-chorusrdb
 		basic_machine=$1
- 		;;
+		;;
 	-hiux*)
 		os=-hiuxwe2
 		;;
+	-sco6)
+		os=-sco5v6
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
 	-sco5)
 		os=-sco3.2v5
 		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
@@ -187,6 +196,10 @@ case $os in
 		# Don't forget version if it is 3.2v4 or newer.
 		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
 		;;
+	-sco5v6*)
+		# Don't forget version if it is 3.2v4 or newer.
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
 	-sco*)
 		os=-sco3.2v2
 		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
@@ -204,6 +217,12 @@ case $os in
 	-isc*)
 		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
 		;;
+	-lynx*178)
+		os=-lynxos178
+		;;
+	-lynx*5)
+		os=-lynxos5
+		;;
 	-lynx*)
 		os=-lynxos
 		;;
@@ -228,64 +247,115 @@ case $basic_machine in
 	# Some are omitted here because they have special meanings below.
 	1750a | 580 \
 	| a29k \
+	| aarch64 | aarch64_be \
 	| alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \
 	| alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \
 	| am33_2.0 \
-	| arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr \
+	| arc | arceb \
+	| arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \
+	| avr | avr32 \
+	| ba \
+	| be32 | be64 \
 	| bfin \
-	| c4x | clipper \
+	| c4x | c8051 | clipper \
 	| d10v | d30v | dlx | dsp16xx \
-	| fr30 | frv \
+	| e2k | epiphany \
+	| fido | fr30 | frv | ft32 \
 	| h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \
+	| hexagon \
 	| i370 | i860 | i960 | ia64 \
 	| ip2k | iq2000 \
-	| m32r | m32rle | m68000 | m68k | m88k | maxq | mcore \
+	| k1om \
+	| le32 | le64 \
+	| lm32 \
+	| m32c | m32r | m32rle | m68000 | m68k | m88k \
+	| maxq | mb | microblaze | microblazeel | mcore | mep | metag \
 	| mips | mipsbe | mipseb | mipsel | mipsle \
 	| mips16 \
 	| mips64 | mips64el \
-	| mips64vr | mips64vrel \
+	| mips64octeon | mips64octeonel \
 	| mips64orion | mips64orionel \
+	| mips64r5900 | mips64r5900el \
+	| mips64vr | mips64vrel \
 	| mips64vr4100 | mips64vr4100el \
 	| mips64vr4300 | mips64vr4300el \
 	| mips64vr5000 | mips64vr5000el \
 	| mips64vr5900 | mips64vr5900el \
 	| mipsisa32 | mipsisa32el \
 	| mipsisa32r2 | mipsisa32r2el \
+	| mipsisa32r6 | mipsisa32r6el \
 	| mipsisa64 | mipsisa64el \
 	| mipsisa64r2 | mipsisa64r2el \
+	| mipsisa64r6 | mipsisa64r6el \
 	| mipsisa64sb1 | mipsisa64sb1el \
 	| mipsisa64sr71k | mipsisa64sr71kel \
+	| mipsr5900 | mipsr5900el \
 	| mipstx39 | mipstx39el \
 	| mn10200 | mn10300 \
-	| ms1 \
+	| moxie \
+	| mt \
 	| msp430 \
+	| nds32 | nds32le | nds32be \
+	| nios | nios2 | nios2eb | nios2el \
 	| ns16k | ns32k \
-	| or32 \
+	| open8 | or1k | or1knd | or32 \
 	| pdp10 | pdp11 | pj | pjl \
-	| powerpc | powerpc64 | powerpc64le | powerpcle | ppcbe \
+	| powerpc | powerpc64 | powerpc64le | powerpcle \
+	| pru \
 	| pyramid \
-	| sh | sh[1234] | sh[24]a | sh[23]e | sh[34]eb | shbe | shle | sh[1234]le | sh3ele \
+	| riscv32 | riscv64 \
+	| rl78 | rx \
+	| score \
+	| sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb | shbe | shle | sh[1234]le | sh3ele \
 	| sh64 | sh64le \
-	| sparc | sparc64 | sparc64b | sparc86x | sparclet | sparclite \
-	| sparcv8 | sparcv9 | sparcv9b \
-	| strongarm \
-	| tahoe | thumb | tic4x | tic80 | tron \
-	| v850 | v850e \
+	| sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \
+	| sparcv8 | sparcv9 | sparcv9b | sparcv9v \
+	| spu \
+	| tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \
+	| ubicom32 \
+	| v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \
+	| visium \
 	| we32k \
-	| x86 | xscale | xscalee[bl] | xstormy16 | xtensa \
-	| z8k)
+	| x86 | xc16x | xstormy16 | xtensa \
+	| z8k | z80)
 		basic_machine=$basic_machine-unknown
 		;;
-	m32c)
-		basic_machine=$basic_machine-unknown
+	c54x)
+		basic_machine=tic54x-unknown
 		;;
-	m6811 | m68hc11 | m6812 | m68hc12)
-		# Motorola 68HC11/12.
+	c55x)
+		basic_machine=tic55x-unknown
+		;;
+	c6x)
+		basic_machine=tic6x-unknown
+		;;
+	leon|leon[3-9])
+		basic_machine=sparc-$basic_machine
+		;;
+	m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip)
 		basic_machine=$basic_machine-unknown
 		os=-none
 		;;
 	m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k)
 		;;
+	ms1)
+		basic_machine=mt-unknown
+		;;
+
+	strongarm | thumb | xscale)
+		basic_machine=arm-unknown
+		;;
+	xgate)
+		basic_machine=$basic_machine-unknown
+		os=-none
+		;;
+	xscaleeb)
+		basic_machine=armeb-unknown
+		;;
+
+	xscaleel)
+		basic_machine=armel-unknown
+		;;
 
 	# We use `pc' rather than `unknown'
 	# because (1) that's what they normally are, and
@@ -301,65 +371,90 @@ case $basic_machine in
 	# Recognize the basic CPU types with company name.
 	580-* \
 	| a29k-* \
+	| aarch64-* | aarch64_be-* \
 	| alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \
 	| alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \
-	| alphapca5[67]-* | alpha64pca5[67]-* | arc-* \
+	| alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \
 	| arm-*  | armbe-* | armle-* | armeb-* | armv*-* \
-	| avr-* \
+	| avr-* | avr32-* \
+	| ba-* \
+	| be32-* | be64-* \
 	| bfin-* | bs2000-* \
-	| c[123]* | c30-* | [cjt]90-* | c4x-* | c54x-* | c55x-* | c6x-* \
-	| clipper-* | craynv-* | cydra-* \
+	| c[123]* | c30-* | [cjt]90-* | c4x-* \
+	| c8051-* | clipper-* | craynv-* | cydra-* \
 	| d10v-* | d30v-* | dlx-* \
-	| elxsi-* \
-	| f30[01]-* | f700-* | fr30-* | frv-* | fx80-* \
+	| e2k-* | elxsi-* \
+	| f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \
 	| h8300-* | h8500-* \
 	| hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \
+	| hexagon-* \
 	| i*86-* | i860-* | i960-* | ia64-* \
 	| ip2k-* | iq2000-* \
-	| m32r-* | m32rle-* \
+	| k1om-* \
+	| le32-* | le64-* \
+	| lm32-* \
+	| m32c-* | m32r-* | m32rle-* \
 	| m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \
-	| m88110-* | m88k-* | maxq-* | mcore-* \
+	| m88110-* | m88k-* | maxq-* | mcore-* | metag-* \
+	| microblaze-* | microblazeel-* \
 	| mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \
 	| mips16-* \
 	| mips64-* | mips64el-* \
-	| mips64vr-* | mips64vrel-* \
+	| mips64octeon-* | mips64octeonel-* \
 	| mips64orion-* | mips64orionel-* \
+	| mips64r5900-* | mips64r5900el-* \
+	| mips64vr-* | mips64vrel-* \
 	| mips64vr4100-* | mips64vr4100el-* \
 	| mips64vr4300-* | mips64vr4300el-* \
 	| mips64vr5000-* | mips64vr5000el-* \
 	| mips64vr5900-* | mips64vr5900el-* \
 	| mipsisa32-* | mipsisa32el-* \
 	| mipsisa32r2-* | mipsisa32r2el-* \
+	| mipsisa32r6-* | mipsisa32r6el-* \
 	| mipsisa64-* | mipsisa64el-* \
 	| mipsisa64r2-* | mipsisa64r2el-* \
+	| mipsisa64r6-* | mipsisa64r6el-* \
 	| mipsisa64sb1-* | mipsisa64sb1el-* \
 	| mipsisa64sr71k-* | mipsisa64sr71kel-* \
+	| mipsr5900-* | mipsr5900el-* \
 	| mipstx39-* | mipstx39el-* \
 	| mmix-* \
-	| ms1-* \
+	| mt-* \
 	| msp430-* \
+	| nds32-* | nds32le-* | nds32be-* \
+	| nios-* | nios2-* | nios2eb-* | nios2el-* \
 	| none-* | np1-* | ns16k-* | ns32k-* \
+	| open8-* \
+	| or1k*-* \
 	| orion-* \
 	| pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \
-	| powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* | ppcbe-* \
+	| powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \
+	| pru-* \
 	| pyramid-* \
-	| romp-* | rs6000-* \
-	| sh-* | sh[1234]-* | sh[24]a-* | sh[23]e-* | sh[34]eb-* | shbe-* \
+	| riscv32-* | riscv64-* \
+	| rl78-* | romp-* | rs6000-* | rx-* \
+	| sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \
 	| shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \
-	| sparc-* | sparc64-* | sparc64b-* | sparc86x-* | sparclet-* \
+	| sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \
 	| sparclite-* \
-	| sparcv8-* | sparcv9-* | sparcv9b-* | strongarm-* | sv1-* | sx?-* \
-	| tahoe-* | thumb-* \
+	| sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \
+	| tahoe-* \
 	| tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \
+	| tile*-* \
 	| tron-* \
-	| v850-* | v850e-* | vax-* \
+	| ubicom32-* \
+	| v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \
+	| vax-* \
+	| visium-* \
 	| we32k-* \
-	| x86-* | x86_64-* | xps100-* | xscale-* | xscalee[bl]-* \
-	| xstormy16-* | xtensa-* \
+	| x86-* | x86_64-* | xc16x-* | xps100-* \
+	| xstormy16-* | xtensa*-* \
 	| ymp-* \
-	| z8k-*)
+	| z8k-* | z80-*)
 		;;
-	m32c-*)
+	# Recognize the basic CPU types without company name, with glob match.
+	xtensa*)
+		basic_machine=$basic_machine-unknown
 		;;
 	# Recognize the various machine names and aliases which stand
 	# for a CPU type and a company and sometimes even an OS.
@@ -377,7 +472,7 @@ case $basic_machine in
 		basic_machine=a29k-amd
 		os=-udi
 		;;
-    	abacus)
+	abacus)
 		basic_machine=abacus-unknown
 		;;
 	adobe68k)
@@ -423,6 +518,13 @@ case $basic_machine in
 		basic_machine=m68k-apollo
 		os=-bsd
 		;;
+	aros)
+		basic_machine=i386-pc
+		os=-aros
+		;;
+	asmjs)
+		basic_machine=asmjs-unknown
+		;;
 	aux)
 		basic_machine=m68k-apple
 		os=-aux
@@ -431,10 +533,35 @@ case $basic_machine in
 		basic_machine=ns32k-sequent
 		os=-dynix
 		;;
+	blackfin)
+		basic_machine=bfin-unknown
+		os=-linux
+		;;
+	blackfin-*)
+		basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'`
+		os=-linux
+		;;
+	bluegene*)
+		basic_machine=powerpc-ibm
+		os=-cnk
+		;;
+	c54x-*)
+		basic_machine=tic54x-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	c55x-*)
+		basic_machine=tic55x-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	c6x-*)
+		basic_machine=tic6x-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
 	c90)
 		basic_machine=c90-cray
 		os=-unicos
 		;;
+	cegcc)
+		basic_machine=arm-unknown
+		os=-cegcc
+		;;
 	convex-c1)
 		basic_machine=c1-convex
 		os=-bsd
@@ -463,8 +590,8 @@ case $basic_machine in
 		basic_machine=craynv-cray
 		os=-unicosmp
 		;;
-	cr16c)
-		basic_machine=cr16c-unknown
+	cr16 | cr16-*)
+		basic_machine=cr16-unknown
 		os=-elf
 		;;
 	crds | unos)
@@ -502,6 +629,10 @@ case $basic_machine in
 		basic_machine=m88k-motorola
 		os=-sysv3
 		;;
+	dicos)
+		basic_machine=i686-pc
+		os=-dicos
+		;;
 	djgpp)
 		basic_machine=i586-pc
 		os=-msdosdjgpp
@@ -514,6 +645,14 @@ case $basic_machine in
 		basic_machine=m68k-bull
 		os=-sysv3
 		;;
+	e500v[12])
+		basic_machine=powerpc-unknown
+		os=$os"spe"
+		;;
+	e500v[12]-*)
+		basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
+		os=$os"spe"
+		;;
 	ebmon29k)
 		basic_machine=a29k-amd
 		os=-ebmon
@@ -617,7 +756,6 @@ case $basic_machine in
 	i370-ibm* | ibm*)
 		basic_machine=i370-ibm
 		;;
-# I'm not sure what "Sysv32" means.  Should this be sysv3.2?
 	i*86v32)
 		basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
 		os=-sysv32
@@ -656,6 +794,17 @@ case $basic_machine in
 		basic_machine=m68k-isi
 		os=-sysv
 		;;
+	leon-*|leon[3-9]-*)
+		basic_machine=sparc-`echo $basic_machine | sed 's/-.*//'`
+		;;
+	m68knommu)
+		basic_machine=m68k-unknown
+		os=-linux
+		;;
+	m68knommu-*)
+		basic_machine=m68k-`echo $basic_machine | sed 's/^[^-]*-//'`
+		os=-linux
+		;;
 	m88k-omron*)
 		basic_machine=m88k-omron
 		;;
@@ -667,10 +816,21 @@ case $basic_machine in
 		basic_machine=ns32k-utek
 		os=-sysv
 		;;
+	microblaze*)
+		basic_machine=microblaze-xilinx
+		;;
+	mingw64)
+		basic_machine=x86_64-pc
+		os=-mingw64
+		;;
 	mingw32)
-		basic_machine=i386-pc
+		basic_machine=i686-pc
 		os=-mingw32
 		;;
+	mingw32ce)
+		basic_machine=arm-unknown
+		os=-mingw32ce
+		;;
 	miniframe)
 		basic_machine=m68000-convergent
 		;;
@@ -692,14 +852,29 @@ case $basic_machine in
 		basic_machine=powerpc-unknown
 		os=-morphos
 		;;
+	moxiebox)
+		basic_machine=moxie-unknown
+		os=-moxiebox
+		;;
 	msdos)
 		basic_machine=i386-pc
 		os=-msdos
 		;;
+	ms1-*)
+		basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'`
+		;;
+	msys)
+		basic_machine=i686-pc
+		os=-msys
+		;;
 	mvs)
 		basic_machine=i370-ibm
 		os=-mvs
 		;;
+	nacl)
+		basic_machine=le32-unknown
+		os=-nacl
+		;;
 	ncr3000)
 		basic_machine=i486-ncr
 		os=-sysv4
@@ -764,6 +939,12 @@ case $basic_machine in
 	np1)
 		basic_machine=np1-gould
 		;;
+	neo-tandem)
+		basic_machine=neo-tandem
+		;;
+	nse-tandem)
+		basic_machine=nse-tandem
+		;;
 	nsr-tandem)
 		basic_machine=nsr-tandem
 		;;
@@ -794,6 +975,14 @@ case $basic_machine in
 		basic_machine=i860-intel
 		os=-osf
 		;;
+	parisc)
+		basic_machine=hppa-unknown
+		os=-linux
+		;;
+	parisc-*)
+		basic_machine=hppa-`echo $basic_machine | sed 's/^[^-]*-//'`
+		os=-linux
+		;;
 	pbd)
 		basic_machine=sparc-tti
 		;;
@@ -803,6 +992,12 @@ case $basic_machine in
 	pc532 | pc532-*)
 		basic_machine=ns32k-pc532
 		;;
+	pc98)
+		basic_machine=i386-pc
+		;;
+	pc98-*)
+		basic_machine=i386-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
 	pentium | p5 | k5 | k6 | nexgen | viac3)
 		basic_machine=i586-pc
 		;;
@@ -832,11 +1027,12 @@ case $basic_machine in
 		;;
 	power)	basic_machine=power-ibm
 		;;
-	ppc)	basic_machine=powerpc-unknown
+	ppc | ppcbe)	basic_machine=powerpc-unknown
 		;;
-	ppc-*)	basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
+	ppc-* | ppcbe-*)
+		basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
 		;;
-	ppcle | powerpclittle | ppc-le | powerpc-little)
+	ppcle | powerpclittle)
 		basic_machine=powerpcle-unknown
 		;;
 	ppcle-* | powerpclittle-*)
@@ -846,7 +1042,7 @@ case $basic_machine in
 		;;
 	ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'`
 		;;
-	ppc64le | powerpc64little | ppc64-le | powerpc64-little)
+	ppc64le | powerpc64little)
 		basic_machine=powerpc64le-unknown
 		;;
 	ppc64le-* | powerpc64little-*)
@@ -859,6 +1055,14 @@ case $basic_machine in
 		basic_machine=i586-unknown
 		os=-pw32
 		;;
+	rdos | rdos64)
+		basic_machine=x86_64-pc
+		os=-rdos
+		;;
+	rdos32)
+		basic_machine=i386-pc
+		os=-rdos
+		;;
 	rom68k)
 		basic_machine=m68k-rom68k
 		os=-coff
@@ -885,6 +1089,10 @@ case $basic_machine in
 	sb1el)
 		basic_machine=mipsisa64sb1el-unknown
 		;;
+	sde)
+		basic_machine=mipsisa32-sde
+		os=-elf
+		;;
 	sei)
 		basic_machine=mips-sei
 		os=-seiux
@@ -896,6 +1104,9 @@ case $basic_machine in
 		basic_machine=sh-hitachi
 		os=-hms
 		;;
+	sh5el)
+		basic_machine=sh5le-unknown
+		;;
 	sh64)
 		basic_machine=sh64-unknown
 		;;
@@ -917,6 +1128,9 @@ case $basic_machine in
 		basic_machine=i860-stratus
 		os=-sysv4
 		;;
+	strongarm-* | thumb-*)
+		basic_machine=arm-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
 	sun2)
 		basic_machine=m68000-sun
 		;;
@@ -973,17 +1187,9 @@ case $basic_machine in
 		basic_machine=t90-cray
 		os=-unicos
 		;;
-	tic54x | c54x*)
-		basic_machine=tic54x-unknown
-		os=-coff
-		;;
-	tic55x | c55x*)
-		basic_machine=tic55x-unknown
-		os=-coff
-		;;
-	tic6x | c6x*)
-		basic_machine=tic6x-unknown
-		os=-coff
+	tile*)
+		basic_machine=$basic_machine-unknown
+		os=-linux-gnu
 		;;
 	tx39)
 		basic_machine=mipstx39-unknown
@@ -1052,6 +1258,9 @@ case $basic_machine in
 	xps | xps100)
 		basic_machine=xps100-honeywell
 		;;
+	xscale-* | xscalee[bl]-*)
+		basic_machine=`echo $basic_machine | sed 's/^xscale/arm/'`
+		;;
 	ymp)
 		basic_machine=ymp-cray
 		os=-unicos
@@ -1060,6 +1269,10 @@ case $basic_machine in
 		basic_machine=z8k-unknown
 		os=-sim
 		;;
+	z80-*-coff)
+		basic_machine=z80-unknown
+		os=-sim
+		;;
 	none)
 		basic_machine=none-none
 		os=-none
@@ -1098,10 +1311,10 @@ case $basic_machine in
 	we32k)
 		basic_machine=we32k-att
 		;;
-	sh[1234] | sh[24]a | sh[34]eb | sh[1234]le | sh[23]ele)
+	sh[1234] | sh[24]a | sh[24]aeb | sh[34]eb | sh[1234]le | sh[23]ele)
 		basic_machine=sh-unknown
 		;;
-	sparc | sparcv8 | sparcv9 | sparcv9b)
+	sparc | sparcv8 | sparcv9 | sparcv9b | sparcv9v)
 		basic_machine=sparc-sun
 		;;
 	cydra)
@@ -1145,9 +1358,12 @@ esac
 if [ x"$os" != x"" ]
 then
 case $os in
-        # First match some system type aliases
-        # that might get confused with valid system types.
+	# First match some system type aliases
+	# that might get confused with valid system types.
 	# -solaris* is a basic system type, with this one exception.
+	-auroraux)
+		os=-auroraux
+		;;
 	-solaris1 | -solaris1.*)
 		os=`echo $os | sed -e 's|solaris1|sunos4|'`
 		;;
@@ -1168,27 +1384,32 @@ case $os in
 	# Each alternative MUST END IN A *, to match a version number.
 	# -sysv* is not here because it comes later, after sysvr4.
 	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
-	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[34]*\
-	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -solaris* | -sym* \
+	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\
+	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \
+	      | -sym* | -kopensolaris* | -plan9* \
 	      | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
-	      | -aos* \
+	      | -aos* | -aros* | -cloudabi* | -sortix* \
 	      | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \
 	      | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \
-	      | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* | -openbsd* \
+	      | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \
+	      | -bitrig* | -openbsd* | -solidbsd* | -libertybsd* \
 	      | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \
 	      | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \
 	      | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \
 	      | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \
-	      | -chorusos* | -chorusrdb* \
-	      | -cygwin* | -pe* | -psos* | -moss* | -proelf* | -rtems* \
-	      | -mingw32* | -linux-gnu* | -linux-uclibc* | -uxpv* | -beos* | -mpeix* | -udk* \
+	      | -chorusos* | -chorusrdb* | -cegcc* | -glidix* \
+	      | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \
+	      | -midipix* | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \
+	      | -linux-newlib* | -linux-musl* | -linux-uclibc* \
+	      | -uxpv* | -beos* | -mpeix* | -udk* | -moxiebox* \
 	      | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \
 	      | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \
 	      | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \
 	      | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \
 	      | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \
 	      | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \
-	      | -skyos* | -haiku*)
+	      | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* \
+	      | -onefs* | -tirtos* | -phoenix* | -fuchsia*)
 	# Remember, each alternative MUST END IN *, to match a version number.
 		;;
 	-qnx*)
@@ -1227,7 +1448,7 @@ case $os in
 	-opened*)
 		os=-openedition
 		;;
-        -os400*)
+	-os400*)
 		os=-os400
 		;;
 	-wince*)
@@ -1276,7 +1497,7 @@ case $os in
 	-sinix*)
 		os=-sysv4
 		;;
-        -tpf*)
+	-tpf*)
 		os=-tpf
 		;;
 	-triton*)
@@ -1312,12 +1533,16 @@ case $os in
 	-aros*)
 		os=-aros
 		;;
-	-kaos*)
-		os=-kaos
-		;;
 	-zvmoe)
 		os=-zvmoe
 		;;
+	-dicos*)
+		os=-dicos
+		;;
+	-nacl*)
+		;;
+	-ios)
+		;;
 	-none)
 		;;
 	*)
@@ -1340,6 +1565,12 @@ else
 # system, and we'll never get to this point.
 
 case $basic_machine in
+	score-*)
+		os=-elf
+		;;
+	spu-*)
+		os=-elf
+		;;
 	*-acorn)
 		os=-riscix1.2
 		;;
@@ -1349,9 +1580,24 @@ case $basic_machine in
 	arm*-semi)
 		os=-aout
 		;;
-    c4x-* | tic4x-*)
-        os=-coff
-        ;;
+	c4x-* | tic4x-*)
+		os=-coff
+		;;
+	c8051-*)
+		os=-elf
+		;;
+	hexagon-*)
+		os=-elf
+		;;
+	tic54x-*)
+		os=-coff
+		;;
+	tic55x-*)
+		os=-coff
+		;;
+	tic6x-*)
+		os=-coff
+		;;
 	# This must come before the *-dec entry.
 	pdp10-*)
 		os=-tops20
@@ -1370,13 +1616,13 @@ case $basic_machine in
 		;;
 	m68000-sun)
 		os=-sunos3
-		# This also exists in the configure program, but was not the
-		# default.
-		# os=-sunos4
 		;;
 	m68*-cisco)
 		os=-aout
 		;;
+	mep-*)
+		os=-elf
+		;;
 	mips*-cisco)
 		os=-elf
 		;;
@@ -1401,7 +1647,7 @@ case $basic_machine in
 	*-ibm)
 		os=-aix
 		;;
-    	*-knuth)
+	*-knuth)
 		os=-mmixware
 		;;
 	*-wec)
@@ -1506,7 +1752,7 @@ case $basic_machine in
 			-sunos*)
 				vendor=sun
 				;;
-			-aix*)
+			-cnk*|-aix*)
 				vendor=ibm
 				;;
 			-beos*)
diff --git a/src/config/configure.in b/src/config/configure.in
index 1b5f4a6..0d1e280 100644
--- a/src/config/configure.in
+++ b/src/config/configure.in
@@ -55,8 +55,8 @@ dnl * Set package information so it only has to be modified in one place
 dnl *********************************************************************
 
 m4_define([M4_HYPRE_NAME],    [hypre])
-m4_define([M4_HYPRE_VERSION], [2.11.1])
-m4_define([M4_HYPRE_DATE],    [2016/06/09])
+m4_define([M4_HYPRE_VERSION], [2.11.2])
+m4_define([M4_HYPRE_DATE],    [2017/03/13])
 m4_define([M4_HYPRE_TIME],    [00:00:00])
 m4_define([M4_HYPRE_BUGS],    [hypre-support at llnl.gov])
 m4_define([M4_HYPRE_SRCDIR],  [`pwd`])
@@ -146,6 +146,10 @@ hypre_using_mli=yes
 hypre_using_openmp=no
 hypre_using_insure=no
 
+hypre_using_caliper=no
+hypre_user_gave_caliper_lib=no
+hypre_user_gave_caliper_inc=no
+
 dnl *********************************************************************
 dnl * Initialize flag-check variables
 dnl *********************************************************************
@@ -765,6 +769,35 @@ AS_HELP_STRING([--with-MPI],
  esac]
 )
 
+AC_ARG_WITH(caliper,
+AS_HELP_STRING([--with-caliper],
+               [Use Caliper instrumentation (default is NO).]),
+               [hypre_using_caliper=yes],
+               [hypre_using_caliper=no])
+
+AS_IF([test "x$with_caliper" = "xyes"],
+      [AC_DEFINE(HYPRE_USING_CALIPER, 1, [Using Caliper instrumentation])],
+      [])
+
+AC_ARG_WITH(caliper-include,
+AS_HELP_STRING([--with-caliper-include=DIR],
+               [Directory where Caliper is installed.]),
+[for caliper_inc_dir in $withval; do
+    CALIPER_INCLUDE="-I$caliper_inc_dir $CALIPER_INCLUDE"
+ done;
+ hypre_user_gave_caliper_inc=yes]
+)
+
+AC_ARG_WITH(caliper-lib,
+AS_HELP_STRING([--with-caliper-lib=LIBS],
+               [LIBS is space-separated linkable list (enclosed in quotes) of libraries
+                needed for Caliper. OK to use -L and -l flags in the list]),
+[for caliper_lib in $withval; do
+    CALIPER_LIBS="$CALIPER_LIBS $caliper_lib"
+ done;
+ hypre_user_gave_caliper_lib=yes]
+)
+
 dnl *********************************************************************
 dnl * Select compilers if not already defined by command line options 
 dnl *********************************************************************
@@ -1194,6 +1227,32 @@ dnl   AC_HYPRE_CHECK_USER_BLASLIBS
 dnl   AC_HYPRE_CHECK_USER_LAPACKLIBS
  fi
 
+
+dnl *********************************************************************
+dnl * Warn if caliper options are incomplete
+dnl *********************************************************************
+if test "$hypre_using_caliper" = "yes"
+then
+   if test "$hypre_user_gave_caliper_inc" != "yes"
+   then
+       AC_MSG_WARN([*******************************************************])
+       AC_MSG_WARN([Configuring with --with-caliper=yes without providing]) 
+       AC_MSG_WARN([--with-caliper-include=<path-to-caliper-install>.])
+       AC_MSG_WARN([Using default user include path.])
+       AC_MSG_WARN([NOTE: Caliper annotations may not work.])
+       AC_MSG_WARN([*******************************************************])
+   fi
+   if test "$hypre_user_gave_caliper_lib" != "yes"
+   then
+       AC_MSG_WARN([*******************************************************])
+       AC_MSG_WARN([Configuring with --with-caliper=yes without providing])
+       AC_MSG_WARN([--with-caliper-lib=<path-to-caliper-lib>.])
+       AC_MSG_WARN([Using default user library path.])
+       AC_MSG_WARN([NOTE: Caliper annotations may not work.])
+       AC_MSG_WARN([*******************************************************])
+   fi
+fi
+
 dnl *********************************************************************
 dnl * Set installation directories
 dnl *********************************************************************
@@ -1296,6 +1355,12 @@ AC_SUBST(LAPACKLIBDIRS)
 AC_SUBST(LAPACKLIBS)
 
 dnl *********************************************************************
+dnl * Caliper instrumentation
+dnl *********************************************************************
+AC_SUBST(CALIPER_INCLUDE)
+AC_SUBST(CALIPER_LIBS)
+
+dnl *********************************************************************
 dnl * ar & ranlib substitution
 dnl *********************************************************************
 AC_SUBST(AR)
diff --git a/src/config/hypre_blas_macros.m4 b/src/config/hypre_blas_macros.m4
index 06d6f01..4dd5b79 100644
--- a/src/config/hypre_blas_macros.m4
+++ b/src/config/hypre_blas_macros.m4
@@ -67,7 +67,7 @@ AC_DEFUN([AC_HYPRE_FIND_BLAS],
 #***************************************************************
   hypre_save_LIBS="$LIBS"
   hypre_save_LDFLGS="$LDFLAGS"
-  LIBS="$LIBS  -nodefaultlibs -L/usr/local/Cellar/gcc/5.3.0/lib/gcc/5/liblgfortran.a -lm"
+  LIBS="$LIBS $FLIBS"
 
 #***************************************************************
 #   Set possible BLAS library names
@@ -77,8 +77,8 @@ AC_DEFUN([AC_HYPRE_FIND_BLAS],
 #***************************************************************
 #   Set search paths for BLAS library
 #***************************************************************
-  temp_FLAGS="-L/usr/local2/lib -L/usr/lib2 -L/lib -L/opt/intel/mkl70/lib/32"
-  LDFLAGS="-nodefaultlibs $temp_FLAGS $LDFLAGS"
+  temp_FLAGS="-L/usr/lib -L/usr/local/lib -L/lib -L/opt/intel/mkl70/lib/32"
+  LDFLAGS="$temp_FLAGS $LDFLAGS"
 
 #***************************************************************
 #   Check for function dgemm in BLAS_LIB_NAMES
diff --git a/src/config/hypre_macros_misc.m4 b/src/config/hypre_macros_misc.m4
index 9bc5838..caadd27 100644
--- a/src/config/hypre_macros_misc.m4
+++ b/src/config/hypre_macros_misc.m4
@@ -116,8 +116,8 @@ then
       icc|mpiicc)
         CFLAGS="-O2"
         if test "$hypre_using_openmp" = "yes" ; then
-          CFLAGS="$CFLAGS -openmp"
-          LDFLAGS="$LDFLAGS -openmp"
+          CFLAGS="$CFLAGS -qopenmp"
+          LDFLAGS="$LDFLAGS -qopenmp"
         fi
         ;;
       pgcc|mpipgcc)
@@ -155,7 +155,7 @@ then
       icpc|icc|mpiicpc|mpiicc)
         CXXFLAGS="-O2"
         if test "$hypre_using_openmp" = "yes" ; then
-          CXXFLAGS="$CXXFLAGS -openmp"
+          CXXFLAGS="$CXXFLAGS -qopenmp"
         fi
         ;;
       pgCC|mpipgCC)
@@ -191,7 +191,7 @@ then
       ifort|mpiifort)
         FFLAGS="-O2"
         if test "$hypre_using_openmp" = "yes" ; then
-          FFLAGS="$FFLAGS -openmp"
+          FFLAGS="$FFLAGS -qopenmp"
         fi
         ;;
       pgf77|mpipgf77)
@@ -236,8 +236,8 @@ then
       icc|mpiicc)
         CFLAGS="-g"
         if test "$hypre_using_openmp" = "yes" ; then
-          CFLAGS="$CFLAGS -openmp"
-          LDFLAGS="$LDFLAGS -openmp"
+          CFLAGS="$CFLAGS -qopenmp"
+          LDFLAGS="$LDFLAGS -qopenmp"
         fi
         ;;
       pgcc|mpipgcc)
@@ -275,7 +275,7 @@ then
       icpc|icc|mpiicpc|mpiicc)
         CXXFLAGS="-g"
         if test "$hypre_using_openmp" = "yes" ; then
-          CXXFLAGS="$CXXFLAGS -openmp"
+          CXXFLAGS="$CXXFLAGS -qopenmp"
         fi
         ;;
       pgCC|mpipgCC)
@@ -311,7 +311,7 @@ then
       ifort|mpiifort)
         FFLAGS="-g"
         if test "$hypre_using_openmp" = "yes" ; then
-          FFLAGS="$FFLAGS -openmp"
+          FFLAGS="$FFLAGS -qopenmp"
         fi
         ;;
       pgf77|mpipgf77)
diff --git a/src/configure b/src/configure
index de8c2f2..c38f578 100755
--- a/src/configure
+++ b/src/configure
@@ -1,7 +1,7 @@
 #! /bin/sh
 # From configure.in Id.
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for hypre 2.11.1.
+# Generated by GNU Autoconf 2.69 for hypre 2.11.2.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -586,8 +586,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='hypre'
 PACKAGE_TARNAME='hypre'
-PACKAGE_VERSION='2.11.1'
-PACKAGE_STRING='hypre 2.11.1'
+PACKAGE_VERSION='2.11.2'
+PACKAGE_STRING='hypre 2.11.2'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -596,6 +596,8 @@ ac_default_prefix=`pwd`/hypre
 ac_subst_vars='LTLIBOBJS
 LIBOBJS
 AR
+CALIPER_LIBS
+CALIPER_INCLUDE
 LAPACKLIBS
 LAPACKLIBDIRS
 HYPRE_LAPACK_FILES
@@ -746,6 +748,9 @@ with_fei
 with_superlu
 with_mli
 with_MPI
+with_caliper
+with_caliper_include
+with_caliper_lib
 with_blas
 with_lapack
 '
@@ -1303,7 +1308,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures hypre 2.11.1 to adapt to many kinds of systems.
+\`configure' configures hypre 2.11.2 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1368,7 +1373,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of hypre 2.11.1:";;
+     short | recursive ) echo "Configuration of hypre 2.11.2:";;
    esac
   cat <<\_ACEOF
 
@@ -1476,6 +1481,12 @@ Optional Packages:
   --with-mli              Use MLI
   --with-MPI              DEFAULT: Compile with MPI. Selecting --without-MPI
                           may affect which compiler is chosen.
+  --with-caliper          Use Caliper instrumentation (default is NO).
+  --with-caliper-include=DIR
+                          Directory where Caliper is installed.
+  --with-caliper-lib=LIBS LIBS is space-separated linkable list (enclosed in
+                          quotes) of libraries needed for Caliper. OK to use
+                          -L and -l flags in the list
   --with-blas             Find a system-provided BLAS library
   --with-lapack           Find a system-provided LAPACK library
 
@@ -1559,7 +1570,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-hypre configure 2.11.1
+hypre configure 2.11.2
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1856,7 +1867,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by hypre $as_me 2.11.1, which was
+It was created by hypre $as_me 2.11.2, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -2242,8 +2253,8 @@ ac_configure="$SHELL $ac_aux_dir/configure"  # Please don't use this var.
 
 
 HYPRE_NAME="hypre"
-HYPRE_VERSION="2.11.1"
-HYPRE_DATE="2016/06/09"
+HYPRE_VERSION="2.11.2"
+HYPRE_DATE="2017/03/13"
 HYPRE_TIME="00:00:00"
 HYPRE_BUGS="hypre-support at llnl.gov"
 HYPRE_SRCDIR="`pwd`"
@@ -2308,6 +2319,10 @@ hypre_using_mli=yes
 hypre_using_openmp=no
 hypre_using_insure=no
 
+hypre_using_caliper=no
+hypre_user_gave_caliper_lib=no
+hypre_user_gave_caliper_inc=no
+
 hypre_blas_lib_old_style=no
 hypre_blas_lib_dir_old_style=no
 hypre_lapack_lib_old_style=no
@@ -3133,6 +3148,43 @@ if test "${with_MPI+set}" = set; then :
 fi
 
 
+
+# Check whether --with-caliper was given.
+if test "${with_caliper+set}" = set; then :
+  withval=$with_caliper; hypre_using_caliper=yes
+else
+  hypre_using_caliper=no
+fi
+
+
+if test "x$with_caliper" = "xyes"; then :
+
+$as_echo "#define HYPRE_USING_CALIPER 1" >>confdefs.h
+
+fi
+
+
+# Check whether --with-caliper-include was given.
+if test "${with_caliper_include+set}" = set; then :
+  withval=$with_caliper_include; for caliper_inc_dir in $withval; do
+    CALIPER_INCLUDE="-I$caliper_inc_dir $CALIPER_INCLUDE"
+ done;
+ hypre_user_gave_caliper_inc=yes
+
+fi
+
+
+
+# Check whether --with-caliper-lib was given.
+if test "${with_caliper_lib+set}" = set; then :
+  withval=$with_caliper_lib; for caliper_lib in $withval; do
+    CALIPER_LIBS="$CALIPER_LIBS $caliper_lib"
+ done;
+ hypre_user_gave_caliper_lib=yes
+
+fi
+
+
 if test "$hypre_user_chose_ccompilers" = "no"
 then
    if test "$hypre_using_mpi" = "no"
@@ -6312,7 +6364,7 @@ fi
 #***************************************************************
   hypre_save_LIBS="$LIBS"
   hypre_save_LDFLGS="$LDFLAGS"
-  LIBS="$LIBS  -nodefaultlibs -L/usr/local/Cellar/gcc/5.3.0/lib/gcc/5/liblgfortran.a -lm"
+  LIBS="$LIBS $FLIBS"
 
 #***************************************************************
 #   Set possible BLAS library names
@@ -6322,8 +6374,8 @@ fi
 #***************************************************************
 #   Set search paths for BLAS library
 #***************************************************************
-  temp_FLAGS="-L/usr/local2/lib -L/usr/lib2 -L/lib -L/opt/intel/mkl70/lib/32"
-  LDFLAGS="-nodefaultlibs $temp_FLAGS $LDFLAGS"
+  temp_FLAGS="-L/usr/lib -L/usr/local/lib -L/lib -L/opt/intel/mkl70/lib/32"
+  LDFLAGS="$temp_FLAGS $LDFLAGS"
 
 #***************************************************************
 #   Check for function dgemm in BLAS_LIB_NAMES
@@ -6769,8 +6821,8 @@ then
       icc|mpiicc)
         CFLAGS="-g"
         if test "$hypre_using_openmp" = "yes" ; then
-          CFLAGS="$CFLAGS -openmp"
-          LDFLAGS="$LDFLAGS -openmp"
+          CFLAGS="$CFLAGS -qopenmp"
+          LDFLAGS="$LDFLAGS -qopenmp"
         fi
         ;;
       pgcc|mpipgcc)
@@ -6808,7 +6860,7 @@ then
       icpc|icc|mpiicpc|mpiicc)
         CXXFLAGS="-g"
         if test "$hypre_using_openmp" = "yes" ; then
-          CXXFLAGS="$CXXFLAGS -openmp"
+          CXXFLAGS="$CXXFLAGS -qopenmp"
         fi
         ;;
       pgCC|mpipgCC)
@@ -6844,7 +6896,7 @@ then
       ifort|mpiifort)
         FFLAGS="-g"
         if test "$hypre_using_openmp" = "yes" ; then
-          FFLAGS="$FFLAGS -openmp"
+          FFLAGS="$FFLAGS -qopenmp"
         fi
         ;;
       pgf77|mpipgf77)
@@ -6882,8 +6934,8 @@ then
       icc|mpiicc)
         CFLAGS="-O2"
         if test "$hypre_using_openmp" = "yes" ; then
-          CFLAGS="$CFLAGS -openmp"
-          LDFLAGS="$LDFLAGS -openmp"
+          CFLAGS="$CFLAGS -qopenmp"
+          LDFLAGS="$LDFLAGS -qopenmp"
         fi
         ;;
       pgcc|mpipgcc)
@@ -6921,7 +6973,7 @@ then
       icpc|icc|mpiicpc|mpiicc)
         CXXFLAGS="-O2"
         if test "$hypre_using_openmp" = "yes" ; then
-          CXXFLAGS="$CXXFLAGS -openmp"
+          CXXFLAGS="$CXXFLAGS -qopenmp"
         fi
         ;;
       pgCC|mpipgCC)
@@ -6957,7 +7009,7 @@ then
       ifort|mpiifort)
         FFLAGS="-O2"
         if test "$hypre_using_openmp" = "yes" ; then
-          FFLAGS="$FFLAGS -openmp"
+          FFLAGS="$FFLAGS -qopenmp"
         fi
         ;;
       pgf77|mpipgf77)
@@ -7145,6 +7197,41 @@ fi
    fi
  fi
 
+
+if test "$hypre_using_caliper" = "yes"
+then
+   if test "$hypre_user_gave_caliper_inc" != "yes"
+   then
+       { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: *******************************************************" >&5
+$as_echo "$as_me: WARNING: *******************************************************" >&2;}
+       { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Configuring with --with-caliper=yes without providing" >&5
+$as_echo "$as_me: WARNING: Configuring with --with-caliper=yes without providing" >&2;}
+       { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --with-caliper-include=<path-to-caliper-install>." >&5
+$as_echo "$as_me: WARNING: --with-caliper-include=<path-to-caliper-install>." >&2;}
+       { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Using default user include path." >&5
+$as_echo "$as_me: WARNING: Using default user include path." >&2;}
+       { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: NOTE: Caliper annotations may not work." >&5
+$as_echo "$as_me: WARNING: NOTE: Caliper annotations may not work." >&2;}
+       { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: *******************************************************" >&5
+$as_echo "$as_me: WARNING: *******************************************************" >&2;}
+   fi
+   if test "$hypre_user_gave_caliper_lib" != "yes"
+   then
+       { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: *******************************************************" >&5
+$as_echo "$as_me: WARNING: *******************************************************" >&2;}
+       { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Configuring with --with-caliper=yes without providing" >&5
+$as_echo "$as_me: WARNING: Configuring with --with-caliper=yes without providing" >&2;}
+       { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --with-caliper-lib=<path-to-caliper-lib>." >&5
+$as_echo "$as_me: WARNING: --with-caliper-lib=<path-to-caliper-lib>." >&2;}
+       { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Using default user library path." >&5
+$as_echo "$as_me: WARNING: Using default user library path." >&2;}
+       { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: NOTE: Caliper annotations may not work." >&5
+$as_echo "$as_me: WARNING: NOTE: Caliper annotations may not work." >&2;}
+       { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: *******************************************************" >&5
+$as_echo "$as_me: WARNING: *******************************************************" >&2;}
+   fi
+fi
+
 HYPRE_INSTALLDIR="${prefix}"
 HYPRE_LIBINSTALL="${libdir}"
 HYPRE_INCINSTALL="${includedir}"
@@ -7320,6 +7407,9 @@ $as_echo "$HYPRE_ARCH" >&6; }
 
 
 
+
+
+
 ac_config_files="$ac_config_files config/Makefile.config"
 
 
@@ -7829,7 +7919,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by hypre $as_me 2.11.1, which was
+This file was extended by hypre $as_me 2.11.2, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -7891,7 +7981,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-hypre config.status 2.11.1
+hypre config.status 2.11.2
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 
diff --git a/src/distributed_ls/Euclid/Factor_dh.c b/src/distributed_ls/Euclid/Factor_dh.c
index 034d4c4..2f024a4 100644
--- a/src/distributed_ls/Euclid/Factor_dh.c
+++ b/src/distributed_ls/Euclid/Factor_dh.c
@@ -39,6 +39,7 @@ static void unadjust_bj_private(Factor_dh mat);
 void Factor_dhCreate(Factor_dh *mat)
 {
   START_FUNC_DH
+  HYPRE_Int i;
   struct _factor_dh* tmp; 
 
   if (np_dh > MAX_MPI_TASKS) {
@@ -74,7 +75,16 @@ void Factor_dhCreate(Factor_dh *mat)
   tmp->numbSolve = NULL;
 
   tmp->debug = Parser_dhHasSwitch(parser_dh, "-debug_Factor");
-
+  
+  /* initialize MPI request to null */
+  for(i=0; i<MAX_MPI_TASKS; i++)
+  {
+     tmp->recv_reqLo[i] = hypre_MPI_REQUEST_NULL;
+     tmp->recv_reqHi[i] = hypre_MPI_REQUEST_NULL;
+     tmp->send_reqLo[i] = hypre_MPI_REQUEST_NULL;
+     tmp->send_reqHi[i] = hypre_MPI_REQUEST_NULL;
+     tmp->requests[i] = hypre_MPI_REQUEST_NULL;
+  }
 /*  Factor_dhZeroTiming(tmp); CHECK_V_ERROR; */
   END_FUNC_DH
 }
@@ -84,6 +94,7 @@ void Factor_dhCreate(Factor_dh *mat)
 void Factor_dhDestroy(Factor_dh mat)
 {
   START_FUNC_DH
+  HYPRE_Int i;
 
   if (mat->rp != NULL) { FREE_DH(mat->rp); CHECK_V_ERROR; }
   if (mat->cval != NULL) { FREE_DH(mat->cval); CHECK_V_ERROR; }
@@ -99,6 +110,16 @@ void Factor_dhDestroy(Factor_dh mat)
   if (mat->sendindHi != NULL) { FREE_DH(mat->sendindHi); CHECK_V_ERROR; }
 
   if (mat->numbSolve != NULL) { Numbering_dhDestroy(mat->numbSolve); CHECK_V_ERROR; }
+  
+  /* cleanup MPI requests */
+  for(i=0; i<MAX_MPI_TASKS; i++)
+  {
+     if(mat->recv_reqLo[i] != hypre_MPI_REQUEST_NULL) hypre_MPI_Request_free(&(mat->recv_reqLo[i]));
+     if(mat->recv_reqHi[i] != hypre_MPI_REQUEST_NULL) hypre_MPI_Request_free(&(mat->recv_reqHi[i]));     
+     if(mat->send_reqLo[i] != hypre_MPI_REQUEST_NULL) hypre_MPI_Request_free(&(mat->send_reqLo[i]));
+     if(mat->send_reqHi[i] != hypre_MPI_REQUEST_NULL) hypre_MPI_Request_free(&(mat->send_reqHi[i]));
+     if(mat->requests[i] != hypre_MPI_REQUEST_NULL) hypre_MPI_Request_free(&(mat->requests[i]));     
+  }
   FREE_DH(mat); CHECK_V_ERROR; 
   END_FUNC_DH
 }
diff --git a/src/distributed_ls/ParaSails/DiagScale.c b/src/distributed_ls/ParaSails/DiagScale.c
index e01ba21..d3dc4e8 100644
--- a/src/distributed_ls/ParaSails/DiagScale.c
+++ b/src/distributed_ls/ParaSails/DiagScale.c
@@ -57,7 +57,7 @@ static void ExchangeDiagEntries(MPI_Comm comm, Matrix *mat, HYPRE_Int reqlen,
     hypre_MPI_Request request;
     HYPRE_Int i, j, this_pe;
 
-    shell_sort(reqlen, reqind);
+    hypre_shell_sort(reqlen, reqind);
 
     *num_requests = 0;
 
diff --git a/src/distributed_ls/ParaSails/Numbering.c b/src/distributed_ls/ParaSails/Numbering.c
index 5470a9a..34a3188 100644
--- a/src/distributed_ls/ParaSails/Numbering.c
+++ b/src/distributed_ls/ParaSails/Numbering.c
@@ -103,7 +103,7 @@ Numbering *NumberingCreate(Matrix *mat, HYPRE_Int size)
     }
 
     /* Sort the indices */
-    shell_sort(num_external, &numb->local_to_global[numb->num_loc]);
+    hypre_shell_sort(num_external, &numb->local_to_global[numb->num_loc]);
 
     /* Redo the hash table for the sorted indices */
     HashReset(numb->hash);
diff --git a/src/distributed_ls/ParaSails/OrderStat.c b/src/distributed_ls/ParaSails/OrderStat.c
index 22a7afa..c9aa53e 100644
--- a/src/distributed_ls/ParaSails/OrderStat.c
+++ b/src/distributed_ls/ParaSails/OrderStat.c
@@ -109,10 +109,10 @@ HYPRE_Real randomized_select(HYPRE_Real *a, HYPRE_Int p, HYPRE_Int r, HYPRE_Int
 }
 
 /*--------------------------------------------------------------------------
- * shell_sort - sorts x[0:n-1] in place, ascending order
+ * hypre_shell_sort - sorts x[0:n-1] in place, ascending order
  *--------------------------------------------------------------------------*/
 
-void shell_sort(const HYPRE_Int n, HYPRE_Int x[])
+void hypre_shell_sort(const HYPRE_Int n, HYPRE_Int x[])
 {
     HYPRE_Int m, max, j, k, itemp;
 
diff --git a/src/distributed_ls/ParaSails/OrderStat.h b/src/distributed_ls/ParaSails/OrderStat.h
index e8455cf..245acf2 100644
--- a/src/distributed_ls/ParaSails/OrderStat.h
+++ b/src/distributed_ls/ParaSails/OrderStat.h
@@ -25,6 +25,6 @@
 #include "_hypre_utilities.h"
 
 HYPRE_Real randomized_select(HYPRE_Real *a, HYPRE_Int p, HYPRE_Int r, HYPRE_Int i);
-void shell_sort(const HYPRE_Int n, HYPRE_Int x[]);
+void hypre_shell_sort(const HYPRE_Int n, HYPRE_Int x[]);
 
 #endif /* _ORDERSTAT_H */
diff --git a/src/distributed_ls/ParaSails/ParaSails.c b/src/distributed_ls/ParaSails/ParaSails.c
index 10f5c92..6aed20d 100644
--- a/src/distributed_ls/ParaSails/ParaSails.c
+++ b/src/distributed_ls/ParaSails/ParaSails.c
@@ -121,7 +121,7 @@ static void SendRequests(MPI_Comm comm, HYPRE_Int tag, Matrix *mat, HYPRE_Int re
     hypre_MPI_Request request;
     HYPRE_Int i, j, this_pe;
 
-    shell_sort(reqlen, reqind);
+    hypre_shell_sort(reqlen, reqind);
 
     *num_requests = 0;
 
diff --git a/src/docs/usr_solvers.tex b/src/docs/usr_solvers.tex
index bda018d..028e639 100644
--- a/src/docs/usr_solvers.tex
+++ b/src/docs/usr_solvers.tex
@@ -409,7 +409,7 @@ Various coarsening techniques are available:
 \begin{itemize}
 \item the Cleary-Luby-Jones-Plassman (CLJP) coarsening,
 \item the Falgout coarsening which is a combination of CLJP and the
-classical RS coarsening algorithm (default),
+classical RS coarsening algorithm,
 \item CGC and CGC-E coarsenings \cite{Griebela_1, Griebel_2},
 \item PMIS and HMIS coarsening algorithms which lead to coarsenings with lower complexities \cite{DeSterck_Yang_Heys_2004}
 as well as
@@ -420,10 +420,12 @@ aggressive coarsening (starting with the finest level)
 via \code{HYPRE_BoomerAMGSetAggNumLevels}. Since aggressive coarsening requires long range
 interpolation, multipass interpolation is always used on levels with aggressive coarsening, unless the user specifies another long-range interpolation suitable for aggressive coarsening.
 
+Note that the default coarsening is HMIS \cite{DeSterck_Yang_Heys_2004}.
+
 \subsection{Interpolation Options}
 Various interpolation techniques can be set using \code{HYPRE_BoomerAMGSetInterpType}:
 \begin{itemize}
-\item the ``classical'' interpolation as defined in \cite{Ruge_Stueben_1987} (default),
+\item the ``classical'' interpolation as defined in \cite{Ruge_Stueben_1987},
 \item direct interpolation \cite{Stueben_1999},
 \item standard interpolation \cite{Stueben_1999},
 \item an extended ``classical'' interpolation, which is a long range interpolation and is recommended to be used with PMIS and HMIS coarsening for harder problems \cite{DeSterck_Falgout_Nolting_Yang_2008},
@@ -439,6 +441,8 @@ and recommended to control complexity and truncate the interpolation operators
 using \code{HYPRE_BoomerAMGSetTruncFactor} and/or \code{HYPRE_BoomerAMGSetPMaxElmts},
 or \code{HYPRE_BoomerAMGSetJacobiTruncTheshold} (for Jacobi interpolation only).
 
+Note that the default interpolation is extended+i interpolation \cite{DeSterck_Falgout_Nolting_Yang_2008} truncated to 4 elements per row.
+
 \subsection{Non-Galerkin Options}
 In order to reduce communication, there is a non-Galerkin coarse grid
 sparsification option available \cite{FaSc2014}.  This option can be used by
@@ -451,7 +455,7 @@ on coarser levels.  A common choice of drop-tolerances is $[0.0,
 first coarse level (level 1), use a drop-tolerance of 0.01 on the second coarse
 level (level 2) and then use 0.05 on all subsequent coarse levels.  While still
 experimental, this capability has significantly improved performance on a
-variety of problems.  See the \code{new_ij} driver for an example usage and the
+variety of problems.  See the \code{ij} driver for an example usage and the
 reference manual for more details.
 
 
diff --git a/src/examples/README_files/ex10.cxx.html b/src/examples/README_files/ex10.cxx.html
index e7f2e9a..ea4de34 100644
--- a/src/examples/README_files/ex10.cxx.html
+++ b/src/examples/README_files/ex10.cxx.html
@@ -140,7 +140,7 @@
 <a name="line133">133</a>       <font color="#2040a0">paramStrings</font><font color="4444FF">[</font><font color="#FF0000">0</font><font color="4444FF">]</font> <font color="4444FF">=</font> <strong>new</strong> <strong>char</strong><font color="4444FF">[</font><font color="#FF0000">100</font><font color="4444FF">]</font><font color="4444FF">;</font>
 <a name="line134">134</a>       <font color="#2040a0">strcpy</font><font color="4444FF">(</font><font color="#2040a0">paramStrings</font><font color="4444FF">[</font><font color="#FF0000">0</font><font color="4444FF">]</font>, <font color="#008000">"externalSolver HYPRE"</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line135">135</a>       <font color="#2040a0">feiPtr</font><font color="4444FF">-</font><font color="4444FF">></font><font color="#2040a0">parameters</font><font color="4444FF">(</font><font color="#FF0000">1</font>, <font color="#2040a0">paramStrings</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line136">136</a>       <strong>delete</strong> <font color="#2040a0">paramStrings</font><font color="4444FF">[</font><font color="#FF0000">0</font><font color="4444FF">]</font><font color="4444FF">;</font>
+<a name="line136">136</a>       <strong>delete</strong> <font color="4444FF">[</font><font color="4444FF">]</font> <font color="#2040a0">paramStrings</font><font color="4444FF">[</font><font color="#FF0000">0</font><font color="4444FF">]</font><font color="4444FF">;</font>
 <a name="line137">137</a>       <strong>delete</strong> <font color="4444FF">[</font><font color="4444FF">]</font> <font color="#2040a0">paramStrings</font><font color="4444FF">;</font>
 <a name="line138">138</a>    <font color="4444FF"><strong>}</strong></font>
 <a name="line139">139</a> 
@@ -456,7 +456,7 @@
 <a name="line449">449</a>       <font color="#2040a0">strcpy</font><font color="4444FF">(</font><font color="#2040a0">paramStrings</font><font color="4444FF">[</font><font color="#FF0000">4</font><font color="4444FF">]</font>, <font color="#008000">"tolerance 1e-6"</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line450">450</a>       <font color="#2040a0">strcpy</font><font color="4444FF">(</font><font color="#2040a0">paramStrings</font><font color="4444FF">[</font><font color="#FF0000">5</font><font color="4444FF">]</font>, <font color="#008000">"gmresDim 30"</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line451">451</a>       <font color="#2040a0">strcpy</font><font color="4444FF">(</font><font color="#2040a0">paramStrings</font><font color="4444FF">[</font><font color="#FF0000">6</font><font color="4444FF">]</font>, <font color="#008000">"amgNumSweeps 1"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line452">452</a>       <font color="#2040a0">strcpy</font><font color="4444FF">(</font><font color="#2040a0">paramStrings</font><font color="4444FF">[</font><font color="#FF0000">7</font><font color="4444FF">]</font>, <font color="#008000">"amgCoarsenType falgout"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line452">452</a>       <font color="#2040a0">strcpy</font><font color="4444FF">(</font><font color="#2040a0">paramStrings</font><font color="4444FF">[</font><font color="#FF0000">7</font><font color="4444FF">]</font>, <font color="#008000">"amgCoarsenType hmis"</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line453">453</a>       <font color="#2040a0">strcpy</font><font color="4444FF">(</font><font color="#2040a0">paramStrings</font><font color="4444FF">[</font><font color="#FF0000">8</font><font color="4444FF">]</font>, <font color="#008000">"amgRelaxType hybridsym"</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line454">454</a>       <font color="#2040a0">strcpy</font><font color="4444FF">(</font><font color="#2040a0">paramStrings</font><font color="4444FF">[</font><font color="#FF0000">9</font><font color="4444FF">]</font>, <font color="#008000">"amgSystemSize 1"</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line455">455</a>       <font color="#2040a0">strcpy</font><font color="4444FF">(</font><font color="#2040a0">paramStrings</font><font color="4444FF">[</font><font color="#FF0000">10</font><font color="4444FF">]</font>, <font color="#008000">"amgStrongThreshold 0.25"</font><font color="4444FF">)</font><font color="4444FF">;</font>
diff --git a/src/examples/README_files/ex11.c.html b/src/examples/README_files/ex11.c.html
index 8aca1e6..7137f06 100644
--- a/src/examples/README_files/ex11.c.html
+++ b/src/examples/README_files/ex11.c.html
@@ -239,124 +239,122 @@
 <a name="line232">232</a>    <font color="4444FF"><strong>{</strong></font>
 <a name="line233">233</a>       <font color="#2040a0">HYPRE_BoomerAMGCreate</font><font color="4444FF">(</font><font color="4444FF">&</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line234">234</a>       <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print amg solution info */</font>
-<a name="line235">235</a>       <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line236">236</a>       <font color="#2040a0">HYPRE_BoomerAMGSetRelaxType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* Sym G.S./Jacobi hybrid */</font>
-<a name="line237">237</a>       <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line238">238</a>       <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">0.0</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance zero */</font>
-<a name="line239">239</a>       <font color="#2040a0">HYPRE_BoomerAMGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* do only one iteration! */</font>
-<a name="line240">240</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line241">241</a> 
-<a name="line242">242</a>    <font color="#444444">/* LOBPCG eigensolver */</font>
-<a name="line243">243</a>    <font color="4444FF"><strong>{</strong></font>
-<a name="line244">244</a>       <strong>int</strong> <font color="#2040a0">time_index</font><font color="4444FF">;</font>
-<a name="line245">245</a> 
-<a name="line246">246</a>       <strong>int</strong> <font color="#2040a0">maxIterations</font> <font color="4444FF">=</font> <font color="#FF0000">100</font><font color="4444FF">;</font> <font color="#444444">/* maximum number of iterations */</font>
-<a name="line247">247</a>       <strong>int</strong> <font color="#2040a0">pcgMode</font> <font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">;</font>         <font color="#444444">/* use rhs as initial guess for inner pcg iterations */</font>
-<a name="line248">248</a>       <strong>int</strong> <font color="#2040a0">verbosity</font> <font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">;</font>       <font color="#444444">/* print iterations info */</font>
-<a name="line249">249</a>       <strong>double</strong> <font color="#2040a0">tol</font> <font color="4444FF">=</font> <font color="#FF0000">1.e-8</font><font color="4444FF">;</font>      <font color="#444444">/* absolute tolerance (all eigenvalues) */</font>
-<a name="line250">250</a>       <strong>int</strong> <font color="#2040a0">lobpcgSeed</font> <font color="4444FF">=</font> <font color="#FF0000">775</font><font color="4444FF">;</font>    <font color="#444444">/* random seed */</font>
-<a name="line251">251</a> 
-<a name="line252">252</a>       <font color="#2040a0">mv_MultiVectorPtr</font> <font color="#2040a0">eigenvectors</font> <font color="4444FF">=</font> <font color="#2040a0">NULL</font><font color="4444FF">;</font>
-<a name="line253">253</a>       <font color="#2040a0">mv_MultiVectorPtr</font> <font color="#2040a0">constraints</font> <font color="4444FF">=</font> <font color="#2040a0">NULL</font><font color="4444FF">;</font>
-<a name="line254">254</a>       <strong>double</strong> <font color="4444FF">*</font><font color="#2040a0">eigenvalues</font> <font color="4444FF">=</font> <font color="#2040a0">NULL</font><font color="4444FF">;</font>
-<a name="line255">255</a> 
-<a name="line256">256</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">!</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
-<a name="line257">257</a>          <font color="#2040a0">verbosity</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font>
-<a name="line258">258</a> 
-<a name="line259">259</a>       <font color="#444444">/* define an interpreter for the ParCSR interface */</font>
-<a name="line260">260</a>       <font color="#2040a0">interpreter</font> <font color="4444FF">=</font> <font color="#2040a0">hypre_CTAlloc</font><font color="4444FF">(</font><font color="#2040a0">mv_InterfaceInterpreter</font>,<font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line261">261</a>       <font color="#2040a0">HYPRE_ParCSRSetupInterpreter</font><font color="4444FF">(</font><font color="#2040a0">interpreter</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line262">262</a>       <font color="#2040a0">HYPRE_ParCSRSetupMatvec</font><font color="4444FF">(</font><font color="4444FF">&</font><font color="#2040a0">matvec_fn</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line263">263</a> 
-<a name="line264">264</a>       <font color="#444444">/* eigenvectors - create a multivector */</font>
-<a name="line265">265</a>       <font color="#2040a0">eigenvectors</font> <font color="4444FF">=</font>
-<a name="line266">266</a>          <font color="#2040a0">mv_MultiVectorCreateFromSampleVector</font><font color="4444FF">(</font><font color="#2040a0">interpreter</font>, <font color="#2040a0">blockSize</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line267">267</a>       <font color="#2040a0">mv_MultiVectorSetRandom</font> <font color="4444FF">(</font><font color="#2040a0">eigenvectors</font>, <font color="#2040a0">lobpcgSeed</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line268">268</a> 
-<a name="line269">269</a>       <font color="#444444">/* eigenvectors - get a pointer */</font>
-<a name="line270">270</a>       <font color="4444FF"><strong>{</strong></font>
-<a name="line271">271</a> 		  <font color="#2040a0">mv_TempMultiVector</font><font color="4444FF">*</font> <font color="#2040a0">tmp</font> <font color="4444FF">=</font> <font color="4444FF">(</font><font color="#2040a0">mv_TempMultiVector</font><font color="4444FF">*</font><font color="4444FF">)</font> <font color="#2040a0">mv_MultiVectorGetData</font><font color="4444FF">(</font><font color="#2040a0">eigenvectors</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line272">272</a>          <font color="#2040a0">pvx</font> <font color="4444FF">=</font> <font color="4444FF">(</font><font color="#2040a0">HYPRE_ParVector</font><font color="4444FF">*</font><font color="4444FF">)</font><font color="4444FF">(</font><font color="#2040a0">tmp</font> <font color="4444FF">-</font><font color="4444FF">></font> <font color="#2040a0">vector</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line273">273</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line274">274</a> 
-<a name="line275">275</a>       <font color="#444444">/* eigenvalues - allocate space */</font>
-<a name="line276">276</a>       <font color="#2040a0">eigenvalues</font> <font color="4444FF">=</font> <font color="4444FF">(</font><strong>double</strong><font color="4444FF">*</font><font color="4444FF">)</font> <font color="#2040a0">calloc</font><font color="4444FF">(</font> <font color="#2040a0">blockSize</font>, <strong>sizeof</strong><font color="4444FF">(</font><strong>double</strong><font color="4444FF">)</font> <font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line277">277</a> 
-<a name="line278">278</a>       <font color="#2040a0">HYPRE_LOBPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">interpreter</font>, <font color="4444FF">&</font><font color="#2040a0">matvec_fn</font>, <font color="4444FF">&</font><font color="#2040a0">lobpcg_solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line279">279</a>       <font color="#2040a0">HYPRE_LOBPCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">lobpcg_solver</font>, <font color="#2040a0">maxIterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line280">280</a>       <font color="#2040a0">HYPRE_LOBPCGSetPrecondUsageMode</font><font color="4444FF">(</font><font color="#2040a0">lobpcg_solver</font>, <font color="#2040a0">pcgMode</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line281">281</a>       <font color="#2040a0">HYPRE_LOBPCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">lobpcg_solver</font>, <font color="#2040a0">tol</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line282">282</a>       <font color="#2040a0">HYPRE_LOBPCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">lobpcg_solver</font>, <font color="#2040a0">verbosity</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line283">283</a> 
-<a name="line284">284</a>       <font color="#444444">/* use a preconditioner */</font>
-<a name="line285">285</a>       <font color="#2040a0">HYPRE_LOBPCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">lobpcg_solver</font>,
-<a name="line286">286</a>                              <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSolve</font>,
-<a name="line287">287</a>                              <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSetup</font>,
-<a name="line288">288</a>                              <font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line289">289</a> 
-<a name="line290">290</a>       <font color="#2040a0">HYPRE_LOBPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">lobpcg_solver</font>, <font color="4444FF">(</font><font color="#2040a0">HYPRE_Matrix</font><font color="4444FF">)</font><font color="#2040a0">parcsr_A</font>,
-<a name="line291">291</a>                         <font color="4444FF">(</font><font color="#2040a0">HYPRE_Vector</font><font color="4444FF">)</font><font color="#2040a0">par_b</font>, <font color="4444FF">(</font><font color="#2040a0">HYPRE_Vector</font><font color="4444FF">)</font><font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line292">292</a> 
-<a name="line293">293</a>       <font color="#2040a0">time_index</font> <font color="4444FF">=</font> <font color="#2040a0">hypre_InitializeTiming</font><font color="4444FF">(</font><font color="#008000">"LOBPCG Solve"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line294">294</a>       <font color="#2040a0">hypre_BeginTiming</font><font color="4444FF">(</font><font color="#2040a0">time_index</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line235">235</a>       <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* 2 sweeps of smoothing */</font>
+<a name="line236">236</a>       <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">0.0</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance zero */</font>
+<a name="line237">237</a>       <font color="#2040a0">HYPRE_BoomerAMGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* do only one iteration! */</font>
+<a name="line238">238</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line239">239</a> 
+<a name="line240">240</a>    <font color="#444444">/* LOBPCG eigensolver */</font>
+<a name="line241">241</a>    <font color="4444FF"><strong>{</strong></font>
+<a name="line242">242</a>       <strong>int</strong> <font color="#2040a0">time_index</font><font color="4444FF">;</font>
+<a name="line243">243</a> 
+<a name="line244">244</a>       <strong>int</strong> <font color="#2040a0">maxIterations</font> <font color="4444FF">=</font> <font color="#FF0000">100</font><font color="4444FF">;</font> <font color="#444444">/* maximum number of iterations */</font>
+<a name="line245">245</a>       <strong>int</strong> <font color="#2040a0">pcgMode</font> <font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">;</font>         <font color="#444444">/* use rhs as initial guess for inner pcg iterations */</font>
+<a name="line246">246</a>       <strong>int</strong> <font color="#2040a0">verbosity</font> <font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">;</font>       <font color="#444444">/* print iterations info */</font>
+<a name="line247">247</a>       <strong>double</strong> <font color="#2040a0">tol</font> <font color="4444FF">=</font> <font color="#FF0000">1.e-8</font><font color="4444FF">;</font>      <font color="#444444">/* absolute tolerance (all eigenvalues) */</font>
+<a name="line248">248</a>       <strong>int</strong> <font color="#2040a0">lobpcgSeed</font> <font color="4444FF">=</font> <font color="#FF0000">775</font><font color="4444FF">;</font>    <font color="#444444">/* random seed */</font>
+<a name="line249">249</a> 
+<a name="line250">250</a>       <font color="#2040a0">mv_MultiVectorPtr</font> <font color="#2040a0">eigenvectors</font> <font color="4444FF">=</font> <font color="#2040a0">NULL</font><font color="4444FF">;</font>
+<a name="line251">251</a>       <font color="#2040a0">mv_MultiVectorPtr</font> <font color="#2040a0">constraints</font> <font color="4444FF">=</font> <font color="#2040a0">NULL</font><font color="4444FF">;</font>
+<a name="line252">252</a>       <strong>double</strong> <font color="4444FF">*</font><font color="#2040a0">eigenvalues</font> <font color="4444FF">=</font> <font color="#2040a0">NULL</font><font color="4444FF">;</font>
+<a name="line253">253</a> 
+<a name="line254">254</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">!</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
+<a name="line255">255</a>          <font color="#2040a0">verbosity</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font>
+<a name="line256">256</a> 
+<a name="line257">257</a>       <font color="#444444">/* define an interpreter for the ParCSR interface */</font>
+<a name="line258">258</a>       <font color="#2040a0">interpreter</font> <font color="4444FF">=</font> <font color="#2040a0">hypre_CTAlloc</font><font color="4444FF">(</font><font color="#2040a0">mv_InterfaceInterpreter</font>,<font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line259">259</a>       <font color="#2040a0">HYPRE_ParCSRSetupInterpreter</font><font color="4444FF">(</font><font color="#2040a0">interpreter</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line260">260</a>       <font color="#2040a0">HYPRE_ParCSRSetupMatvec</font><font color="4444FF">(</font><font color="4444FF">&</font><font color="#2040a0">matvec_fn</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line261">261</a> 
+<a name="line262">262</a>       <font color="#444444">/* eigenvectors - create a multivector */</font>
+<a name="line263">263</a>       <font color="#2040a0">eigenvectors</font> <font color="4444FF">=</font>
+<a name="line264">264</a>          <font color="#2040a0">mv_MultiVectorCreateFromSampleVector</font><font color="4444FF">(</font><font color="#2040a0">interpreter</font>, <font color="#2040a0">blockSize</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line265">265</a>       <font color="#2040a0">mv_MultiVectorSetRandom</font> <font color="4444FF">(</font><font color="#2040a0">eigenvectors</font>, <font color="#2040a0">lobpcgSeed</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line266">266</a> 
+<a name="line267">267</a>       <font color="#444444">/* eigenvectors - get a pointer */</font>
+<a name="line268">268</a>       <font color="4444FF"><strong>{</strong></font>
+<a name="line269">269</a> 		  <font color="#2040a0">mv_TempMultiVector</font><font color="4444FF">*</font> <font color="#2040a0">tmp</font> <font color="4444FF">=</font> <font color="4444FF">(</font><font color="#2040a0">mv_TempMultiVector</font><font color="4444FF">*</font><font color="4444FF">)</font> <font color="#2040a0">mv_MultiVectorGetData</font><font color="4444FF">(</font><font color="#2040a0">eigenvectors</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line270">270</a>          <font color="#2040a0">pvx</font> <font color="4444FF">=</font> <font color="4444FF">(</font><font color="#2040a0">HYPRE_ParVector</font><font color="4444FF">*</font><font color="4444FF">)</font><font color="4444FF">(</font><font color="#2040a0">tmp</font> <font color="4444FF">-</font><font color="4444FF">></font> <font color="#2040a0">vector</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line271">271</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line272">272</a> 
+<a name="line273">273</a>       <font color="#444444">/* eigenvalues - allocate space */</font>
+<a name="line274">274</a>       <font color="#2040a0">eigenvalues</font> <font color="4444FF">=</font> <font color="4444FF">(</font><strong>double</strong><font color="4444FF">*</font><font color="4444FF">)</font> <font color="#2040a0">calloc</font><font color="4444FF">(</font> <font color="#2040a0">blockSize</font>, <strong>sizeof</strong><font color="4444FF">(</font><strong>double</strong><font color="4444FF">)</font> <font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line275">275</a> 
+<a name="line276">276</a>       <font color="#2040a0">HYPRE_LOBPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">interpreter</font>, <font color="4444FF">&</font><font color="#2040a0">matvec_fn</font>, <font color="4444FF">&</font><font color="#2040a0">lobpcg_solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line277">277</a>       <font color="#2040a0">HYPRE_LOBPCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">lobpcg_solver</font>, <font color="#2040a0">maxIterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line278">278</a>       <font color="#2040a0">HYPRE_LOBPCGSetPrecondUsageMode</font><font color="4444FF">(</font><font color="#2040a0">lobpcg_solver</font>, <font color="#2040a0">pcgMode</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line279">279</a>       <font color="#2040a0">HYPRE_LOBPCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">lobpcg_solver</font>, <font color="#2040a0">tol</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line280">280</a>       <font color="#2040a0">HYPRE_LOBPCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">lobpcg_solver</font>, <font color="#2040a0">verbosity</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line281">281</a> 
+<a name="line282">282</a>       <font color="#444444">/* use a preconditioner */</font>
+<a name="line283">283</a>       <font color="#2040a0">HYPRE_LOBPCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">lobpcg_solver</font>,
+<a name="line284">284</a>                              <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSolve</font>,
+<a name="line285">285</a>                              <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSetup</font>,
+<a name="line286">286</a>                              <font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line287">287</a> 
+<a name="line288">288</a>       <font color="#2040a0">HYPRE_LOBPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">lobpcg_solver</font>, <font color="4444FF">(</font><font color="#2040a0">HYPRE_Matrix</font><font color="4444FF">)</font><font color="#2040a0">parcsr_A</font>,
+<a name="line289">289</a>                         <font color="4444FF">(</font><font color="#2040a0">HYPRE_Vector</font><font color="4444FF">)</font><font color="#2040a0">par_b</font>, <font color="4444FF">(</font><font color="#2040a0">HYPRE_Vector</font><font color="4444FF">)</font><font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line290">290</a> 
+<a name="line291">291</a>       <font color="#2040a0">time_index</font> <font color="4444FF">=</font> <font color="#2040a0">hypre_InitializeTiming</font><font color="4444FF">(</font><font color="#008000">"LOBPCG Solve"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line292">292</a>       <font color="#2040a0">hypre_BeginTiming</font><font color="4444FF">(</font><font color="#2040a0">time_index</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line293">293</a> 
+<a name="line294">294</a>       <font color="#2040a0">HYPRE_LOBPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">lobpcg_solver</font>, <font color="#2040a0">constraints</font>, <font color="#2040a0">eigenvectors</font>, <font color="#2040a0">eigenvalues</font> <font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line295">295</a> 
-<a name="line296">296</a>       <font color="#2040a0">HYPRE_LOBPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">lobpcg_solver</font>, <font color="#2040a0">constraints</font>, <font color="#2040a0">eigenvectors</font>, <font color="#2040a0">eigenvalues</font> <font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line297">297</a> 
-<a name="line298">298</a>       <font color="#2040a0">hypre_EndTiming</font><font color="4444FF">(</font><font color="#2040a0">time_index</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line299">299</a>       <font color="#2040a0">hypre_PrintTiming</font><font color="4444FF">(</font><font color="#008000">"Solve phase times"</font>, <font color="#2040a0">MPI_COMM_WORLD</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line300">300</a>       <font color="#2040a0">hypre_FinalizeTiming</font><font color="4444FF">(</font><font color="#2040a0">time_index</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line301">301</a>       <font color="#2040a0">hypre_ClearTiming</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line302">302</a> 
-<a name="line303">303</a>       <font color="#444444">/* clean-up */</font>
-<a name="line304">304</a>       <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line305">305</a>       <font color="#2040a0">HYPRE_LOBPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">lobpcg_solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line306">306</a>       <font color="#2040a0">hypre_TFree</font><font color="4444FF">(</font><font color="#2040a0">eigenvalues</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line307">307</a>       <font color="#2040a0">hypre_TFree</font><font color="4444FF">(</font><font color="#2040a0">interpreter</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line308">308</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line309">309</a> 
-<a name="line310">310</a>    <font color="#444444">/* Save the solution for GLVis visualization, see vis/glvis-ex11.sh */</font>
-<a name="line311">311</a>    <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">vis</font><font color="4444FF">)</font>
-<a name="line312">312</a>    <font color="4444FF"><strong>{</strong></font>
-<a name="line313">313</a>       <font color="#2040a0">FILE</font> <font color="4444FF">*</font><font color="#2040a0">file</font><font color="4444FF">;</font>
-<a name="line314">314</a>       <strong>char</strong> <font color="#2040a0">filename</font><font color="4444FF">[</font><font color="#FF0000">255</font><font color="4444FF">]</font><font color="4444FF">;</font>
-<a name="line315">315</a> 
-<a name="line316">316</a>       <strong>int</strong> <font color="#2040a0">nvalues</font> <font color="4444FF">=</font> <font color="#2040a0">local_size</font><font color="4444FF">;</font>
-<a name="line317">317</a>       <strong>double</strong> <font color="4444FF">*</font><font color="#2040a0">values</font><font color="4444FF">;</font>
-<a name="line318">318</a> 
-<a name="line319">319</a>       <font color="#444444">/* get the local solution */</font>
-<a name="line320">320</a>       <font color="#2040a0">values</font> <font color="4444FF">=</font> <font color="#2040a0">hypre_VectorData</font><font color="4444FF">(</font><font color="#2040a0">hypre_ParVectorLocalVector</font><font color="4444FF">(</font>
-<a name="line321">321</a>                                    <font color="4444FF">(</font><font color="#2040a0">hypre_ParVector</font><font color="4444FF">*</font><font color="4444FF">)</font><font color="#2040a0">pvx</font><font color="4444FF">[</font><font color="#2040a0">blockSize</font><font color="4444FF">-</font><font color="#FF0000">1</font><font color="4444FF">]</font><font color="4444FF">)</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line322">322</a> 
-<a name="line323">323</a>       <font color="#2040a0">sprintf</font><font color="4444FF">(</font><font color="#2040a0">filename</font>, <font color="#008000">"%s.%06d"</font>, <font color="#008000">"vis/ex11.sol"</font>, <font color="#2040a0">myid</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line324">324</a>       <strong>if</strong> <font color="4444FF">(</font><font color="4444FF">(</font><font color="#2040a0">file</font> <font color="4444FF">=</font> <font color="#2040a0">fopen</font><font color="4444FF">(</font><font color="#2040a0">filename</font>, <font color="#008000">"w"</font><font color="4444FF">)</font><font color="4444FF">)</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#2040a0">NULL</font><font color="4444FF">)</font>
-<a name="line325">325</a>       <font color="4444FF"><strong>{</strong></font>
-<a name="line326">326</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Error: can't open output file %s<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">filename</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line327">327</a>          <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line328">328</a>          <font color="#2040a0">exit</font><font color="4444FF">(</font><font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line329">329</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line330">330</a> 
-<a name="line331">331</a>       <font color="#444444">/* save solution */</font>
-<a name="line332">332</a>       <strong>for</strong> <font color="4444FF">(</font><font color="#2040a0">i</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font> <font color="#2040a0">i</font> <font color="4444FF"><</font> <font color="#2040a0">nvalues</font><font color="4444FF">;</font> <font color="#2040a0">i</font><font color="4444FF">+</font><font color="4444FF">+</font><font color="4444FF">)</font>
-<a name="line333">333</a>          <font color="#2040a0">fprintf</font><font color="4444FF">(</font><font color="#2040a0">file</font>, <font color="#008000">"%.14e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">values</font><font color="4444FF">[</font><font color="#2040a0">i</font><font color="4444FF">]</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line334">334</a> 
-<a name="line335">335</a>       <font color="#2040a0">fflush</font><font color="4444FF">(</font><font color="#2040a0">file</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line336">336</a>       <font color="#2040a0">fclose</font><font color="4444FF">(</font><font color="#2040a0">file</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line337">337</a> 
-<a name="line338">338</a>       <font color="#444444">/* save global finite element mesh */</font>
-<a name="line339">339</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
-<a name="line340">340</a>          <font color="#2040a0">GLVis_PrintGlobalSquareMesh</font><font color="4444FF">(</font><font color="#008000">"vis/ex11.mesh"</font>, <font color="#2040a0">n</font><font color="4444FF">-</font><font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line341">341</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line342">342</a> 
-<a name="line343">343</a>    <font color="#444444">/* Clean up */</font>
-<a name="line344">344</a>    <font color="#2040a0">HYPRE_IJMatrixDestroy</font><font color="4444FF">(</font><font color="#2040a0">A</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line345">345</a>    <font color="#2040a0">HYPRE_IJVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">b</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line346">346</a>    <font color="#2040a0">HYPRE_IJVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line347">347</a> 
-<a name="line348">348</a>    <font color="#444444">/* Finalize MPI*/</font>
-<a name="line349">349</a>    <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line350">350</a> 
-<a name="line351">351</a>    <strong>return</strong><font color="4444FF">(</font><font color="#FF0000">0</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line352">352</a> <font color="4444FF"><strong>}</strong></font>
+<a name="line296">296</a>       <font color="#2040a0">hypre_EndTiming</font><font color="4444FF">(</font><font color="#2040a0">time_index</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line297">297</a>       <font color="#2040a0">hypre_PrintTiming</font><font color="4444FF">(</font><font color="#008000">"Solve phase times"</font>, <font color="#2040a0">MPI_COMM_WORLD</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line298">298</a>       <font color="#2040a0">hypre_FinalizeTiming</font><font color="4444FF">(</font><font color="#2040a0">time_index</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line299">299</a>       <font color="#2040a0">hypre_ClearTiming</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line300">300</a> 
+<a name="line301">301</a>       <font color="#444444">/* clean-up */</font>
+<a name="line302">302</a>       <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line303">303</a>       <font color="#2040a0">HYPRE_LOBPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">lobpcg_solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line304">304</a>       <font color="#2040a0">hypre_TFree</font><font color="4444FF">(</font><font color="#2040a0">eigenvalues</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line305">305</a>       <font color="#2040a0">hypre_TFree</font><font color="4444FF">(</font><font color="#2040a0">interpreter</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line306">306</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line307">307</a> 
+<a name="line308">308</a>    <font color="#444444">/* Save the solution for GLVis visualization, see vis/glvis-ex11.sh */</font>
+<a name="line309">309</a>    <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">vis</font><font color="4444FF">)</font>
+<a name="line310">310</a>    <font color="4444FF"><strong>{</strong></font>
+<a name="line311">311</a>       <font color="#2040a0">FILE</font> <font color="4444FF">*</font><font color="#2040a0">file</font><font color="4444FF">;</font>
+<a name="line312">312</a>       <strong>char</strong> <font color="#2040a0">filename</font><font color="4444FF">[</font><font color="#FF0000">255</font><font color="4444FF">]</font><font color="4444FF">;</font>
+<a name="line313">313</a> 
+<a name="line314">314</a>       <strong>int</strong> <font color="#2040a0">nvalues</font> <font color="4444FF">=</font> <font color="#2040a0">local_size</font><font color="4444FF">;</font>
+<a name="line315">315</a>       <strong>double</strong> <font color="4444FF">*</font><font color="#2040a0">values</font><font color="4444FF">;</font>
+<a name="line316">316</a> 
+<a name="line317">317</a>       <font color="#444444">/* get the local solution */</font>
+<a name="line318">318</a>       <font color="#2040a0">values</font> <font color="4444FF">=</font> <font color="#2040a0">hypre_VectorData</font><font color="4444FF">(</font><font color="#2040a0">hypre_ParVectorLocalVector</font><font color="4444FF">(</font>
+<a name="line319">319</a>                                    <font color="4444FF">(</font><font color="#2040a0">hypre_ParVector</font><font color="4444FF">*</font><font color="4444FF">)</font><font color="#2040a0">pvx</font><font color="4444FF">[</font><font color="#2040a0">blockSize</font><font color="4444FF">-</font><font color="#FF0000">1</font><font color="4444FF">]</font><font color="4444FF">)</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line320">320</a> 
+<a name="line321">321</a>       <font color="#2040a0">sprintf</font><font color="4444FF">(</font><font color="#2040a0">filename</font>, <font color="#008000">"%s.%06d"</font>, <font color="#008000">"vis/ex11.sol"</font>, <font color="#2040a0">myid</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line322">322</a>       <strong>if</strong> <font color="4444FF">(</font><font color="4444FF">(</font><font color="#2040a0">file</font> <font color="4444FF">=</font> <font color="#2040a0">fopen</font><font color="4444FF">(</font><font color="#2040a0">filename</font>, <font color="#008000">"w"</font><font color="4444FF">)</font><font color="4444FF">)</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#2040a0">NULL</font><font color="4444FF">)</font>
+<a name="line323">323</a>       <font color="4444FF"><strong>{</strong></font>
+<a name="line324">324</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Error: can't open output file %s<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">filename</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line325">325</a>          <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line326">326</a>          <font color="#2040a0">exit</font><font color="4444FF">(</font><font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line327">327</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line328">328</a> 
+<a name="line329">329</a>       <font color="#444444">/* save solution */</font>
+<a name="line330">330</a>       <strong>for</strong> <font color="4444FF">(</font><font color="#2040a0">i</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font> <font color="#2040a0">i</font> <font color="4444FF"><</font> <font color="#2040a0">nvalues</font><font color="4444FF">;</font> <font color="#2040a0">i</font><font color="4444FF">+</font><font color="4444FF">+</font><font color="4444FF">)</font>
+<a name="line331">331</a>          <font color="#2040a0">fprintf</font><font color="4444FF">(</font><font color="#2040a0">file</font>, <font color="#008000">"%.14e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">values</font><font color="4444FF">[</font><font color="#2040a0">i</font><font color="4444FF">]</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line332">332</a> 
+<a name="line333">333</a>       <font color="#2040a0">fflush</font><font color="4444FF">(</font><font color="#2040a0">file</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line334">334</a>       <font color="#2040a0">fclose</font><font color="4444FF">(</font><font color="#2040a0">file</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line335">335</a> 
+<a name="line336">336</a>       <font color="#444444">/* save global finite element mesh */</font>
+<a name="line337">337</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
+<a name="line338">338</a>          <font color="#2040a0">GLVis_PrintGlobalSquareMesh</font><font color="4444FF">(</font><font color="#008000">"vis/ex11.mesh"</font>, <font color="#2040a0">n</font><font color="4444FF">-</font><font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line339">339</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line340">340</a> 
+<a name="line341">341</a>    <font color="#444444">/* Clean up */</font>
+<a name="line342">342</a>    <font color="#2040a0">HYPRE_IJMatrixDestroy</font><font color="4444FF">(</font><font color="#2040a0">A</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line343">343</a>    <font color="#2040a0">HYPRE_IJVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">b</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line344">344</a>    <font color="#2040a0">HYPRE_IJVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line345">345</a> 
+<a name="line346">346</a>    <font color="#444444">/* Finalize MPI*/</font>
+<a name="line347">347</a>    <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line348">348</a> 
+<a name="line349">349</a>    <strong>return</strong><font color="4444FF">(</font><font color="#FF0000">0</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line350">350</a> <font color="4444FF"><strong>}</strong></font>
 
 </pre>
 <hr>
diff --git a/src/examples/README_files/ex12.c.html b/src/examples/README_files/ex12.c.html
index aeda282..b0c6f0b 100644
--- a/src/examples/README_files/ex12.c.html
+++ b/src/examples/README_files/ex12.c.html
@@ -456,45 +456,44 @@
 <a name="line449">449</a>       <font color="#2040a0">HYPRE_BoomerAMGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line450">450</a>       <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">0.0</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line451">451</a>       <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print amg solution info */</font>
-<a name="line452">452</a>       <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line453">453</a>       <font color="#2040a0">HYPRE_BoomerAMGSetRelaxType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* Sym G.S./Jacobi hybrid */</font>
-<a name="line454">454</a>       <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line455">455</a> 
-<a name="line456">456</a>       <font color="#444444">/* Set preconditioner and solve */</font>
-<a name="line457">457</a>       <font color="#2040a0">HYPRE_ParCSRPCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">HYPRE_BoomerAMGSolve</font>,
-<a name="line458">458</a>                                 <font color="#2040a0">HYPRE_BoomerAMGSetup</font>, <font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line459">459</a>       <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parA</font>, <font color="#2040a0">parb</font>, <font color="#2040a0">parx</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line460">460</a>       <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parA</font>, <font color="#2040a0">parb</font>, <font color="#2040a0">parx</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line461">461</a> 
-<a name="line462">462</a>       <font color="#444444">/* Free memory */</font>
-<a name="line463">463</a>       <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line464">464</a>       <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line465">465</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line466">466</a> 
-<a name="line467">467</a>    <font color="#444444">/* Save the solution for GLVis visualization, see vis/glvis-ex12.sh */</font>
-<a name="line468">468</a>    <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">vis</font><font color="4444FF">)</font>
-<a name="line469">469</a>    <font color="4444FF"><strong>{</strong></font>
-<a name="line470">470</a>       <font color="#444444">/* Gather the solution vector */</font>
-<a name="line471">471</a>       <font color="#2040a0">HYPRE_SStructVectorGather</font><font color="4444FF">(</font><font color="#2040a0">x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line472">472</a> 
-<a name="line473">473</a>       <font color="#2040a0">GLVis_PrintSStructGrid</font><font color="4444FF">(</font><font color="#2040a0">grid</font>, <font color="#008000">"vis/ex12.mesh"</font>, <font color="#2040a0">myid</font>, <font color="#2040a0">NULL</font>, <font color="#2040a0">NULL</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line474">474</a>       <font color="#2040a0">GLVis_PrintSStructVector</font><font color="4444FF">(</font><font color="#2040a0">x</font>, <font color="#FF0000">0</font>, <font color="#008000">"vis/ex12.sol"</font>, <font color="#2040a0">myid</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line475">475</a>       <font color="#2040a0">GLVis_PrintData</font><font color="4444FF">(</font><font color="#008000">"vis/ex12.data"</font>, <font color="#2040a0">myid</font>, <font color="#2040a0">num_procs</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line476">476</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line477">477</a> 
-<a name="line478">478</a>    <font color="#444444">/* Free memory */</font>
-<a name="line479">479</a>    <font color="#2040a0">HYPRE_SStructGridDestroy</font><font color="4444FF">(</font><font color="#2040a0">grid</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line480">480</a>    <font color="#2040a0">HYPRE_SStructStencilDestroy</font><font color="4444FF">(</font><font color="#2040a0">stencil</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line481">481</a>    <font color="#2040a0">HYPRE_SStructGraphDestroy</font><font color="4444FF">(</font><font color="#2040a0">graph</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line482">482</a>    <font color="#2040a0">HYPRE_SStructMatrixDestroy</font><font color="4444FF">(</font><font color="#2040a0">A</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line483">483</a>    <font color="#2040a0">HYPRE_SStructVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">b</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line484">484</a>    <font color="#2040a0">HYPRE_SStructVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line485">485</a> 
-<a name="line486">486</a>    <font color="#444444">/* Finalize MPI */</font>
-<a name="line487">487</a>    <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line488">488</a> 
-<a name="line489">489</a>    <strong>return</strong> <font color="4444FF">(</font><font color="#FF0000">0</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line490">490</a> <font color="4444FF"><strong>}</strong></font>
+<a name="line452">452</a>       <font color="#2040a0">HYPRE_BoomerAMGSetRelaxType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* Sym G.S./Jacobi hybrid */</font>
+<a name="line453">453</a>       <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line454">454</a> 
+<a name="line455">455</a>       <font color="#444444">/* Set preconditioner and solve */</font>
+<a name="line456">456</a>       <font color="#2040a0">HYPRE_ParCSRPCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">HYPRE_BoomerAMGSolve</font>,
+<a name="line457">457</a>                                 <font color="#2040a0">HYPRE_BoomerAMGSetup</font>, <font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line458">458</a>       <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parA</font>, <font color="#2040a0">parb</font>, <font color="#2040a0">parx</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line459">459</a>       <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parA</font>, <font color="#2040a0">parb</font>, <font color="#2040a0">parx</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line460">460</a> 
+<a name="line461">461</a>       <font color="#444444">/* Free memory */</font>
+<a name="line462">462</a>       <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line463">463</a>       <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line464">464</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line465">465</a> 
+<a name="line466">466</a>    <font color="#444444">/* Save the solution for GLVis visualization, see vis/glvis-ex12.sh */</font>
+<a name="line467">467</a>    <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">vis</font><font color="4444FF">)</font>
+<a name="line468">468</a>    <font color="4444FF"><strong>{</strong></font>
+<a name="line469">469</a>       <font color="#444444">/* Gather the solution vector */</font>
+<a name="line470">470</a>       <font color="#2040a0">HYPRE_SStructVectorGather</font><font color="4444FF">(</font><font color="#2040a0">x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line471">471</a> 
+<a name="line472">472</a>       <font color="#2040a0">GLVis_PrintSStructGrid</font><font color="4444FF">(</font><font color="#2040a0">grid</font>, <font color="#008000">"vis/ex12.mesh"</font>, <font color="#2040a0">myid</font>, <font color="#2040a0">NULL</font>, <font color="#2040a0">NULL</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line473">473</a>       <font color="#2040a0">GLVis_PrintSStructVector</font><font color="4444FF">(</font><font color="#2040a0">x</font>, <font color="#FF0000">0</font>, <font color="#008000">"vis/ex12.sol"</font>, <font color="#2040a0">myid</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line474">474</a>       <font color="#2040a0">GLVis_PrintData</font><font color="4444FF">(</font><font color="#008000">"vis/ex12.data"</font>, <font color="#2040a0">myid</font>, <font color="#2040a0">num_procs</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line475">475</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line476">476</a> 
+<a name="line477">477</a>    <font color="#444444">/* Free memory */</font>
+<a name="line478">478</a>    <font color="#2040a0">HYPRE_SStructGridDestroy</font><font color="4444FF">(</font><font color="#2040a0">grid</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line479">479</a>    <font color="#2040a0">HYPRE_SStructStencilDestroy</font><font color="4444FF">(</font><font color="#2040a0">stencil</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line480">480</a>    <font color="#2040a0">HYPRE_SStructGraphDestroy</font><font color="4444FF">(</font><font color="#2040a0">graph</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line481">481</a>    <font color="#2040a0">HYPRE_SStructMatrixDestroy</font><font color="4444FF">(</font><font color="#2040a0">A</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line482">482</a>    <font color="#2040a0">HYPRE_SStructVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">b</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line483">483</a>    <font color="#2040a0">HYPRE_SStructVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line484">484</a> 
+<a name="line485">485</a>    <font color="#444444">/* Finalize MPI */</font>
+<a name="line486">486</a>    <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line487">487</a> 
+<a name="line488">488</a>    <strong>return</strong> <font color="4444FF">(</font><font color="#FF0000">0</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line489">489</a> <font color="4444FF"><strong>}</strong></font>
 
 </pre>
 <hr>
diff --git a/src/examples/README_files/ex12f.f.html b/src/examples/README_files/ex12f.f.html
index fecc099..5eae290 100644
--- a/src/examples/README_files/ex12f.f.html
+++ b/src/examples/README_files/ex12f.f.html
@@ -456,33 +456,34 @@
 <a name="line449">449</a> <font color="#444444">c        Print amg solution info</font>
 <a name="line450">450</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
 <a name="line451">451</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line452">452</a> <font color="#444444">c        Sym G.S./Jacobi hybrid</font>
-<a name="line453">453</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetRelaxType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line454">454</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line455">455</a> <font color="#444444">c        Set preconditioner (BoomerAMG = 2) and solve</font>
-<a name="line456">456</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font>, <font color="#2040a0">precond</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line457">457</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parA</font>, <font color="#2040a0">parb</font>, <font color="#2040a0">parx</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line458">458</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parA</font>, <font color="#2040a0">parb</font>, <font color="#2040a0">parx</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line459">459</a> 
-<a name="line460">460</a> <font color="#444444">c        Free memory</font>
-<a name="line461">461</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line462">462</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line463">463</a> 
-<a name="line464">464</a>       <strong>endif</strong>
-<a name="line465">465</a> 
-<a name="line466">466</a> <font color="#444444">c     Free memory</font>
-<a name="line467">467</a>       <strong>call</strong> <font color="#2040a0">HYPRE_SStructGridDestroy</font><font color="4444FF">(</font><font color="#2040a0">grid</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line468">468</a>       <strong>call</strong> <font color="#2040a0">HYPRE_SStructStencilDestroy</font><font color="4444FF">(</font><font color="#2040a0">stencil</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line469">469</a>       <strong>call</strong> <font color="#2040a0">HYPRE_SStructGraphDestroy</font><font color="4444FF">(</font><font color="#2040a0">graph</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line470">470</a>       <strong>call</strong> <font color="#2040a0">HYPRE_SStructMatrixDestroy</font><font color="4444FF">(</font><font color="#2040a0">A</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line471">471</a>       <strong>call</strong> <font color="#2040a0">HYPRE_SStructVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">b</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line472">472</a>       <strong>call</strong> <font color="#2040a0">HYPRE_SStructVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">x</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line473">473</a> 
-<a name="line474">474</a> <font color="#444444">c     Finalize MPI</font>
-<a name="line475">475</a>       <strong>call</strong> <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line476">476</a> 
-<a name="line477">477</a>       <strong>stop</strong>
-<a name="line478">478</a>       <strong>end</strong>
+<a name="line452">452</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetOldDefault</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line453">453</a> <font color="#444444">c        Sym G.S./Jacobi hybrid</font>
+<a name="line454">454</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetRelaxType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line455">455</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line456">456</a> <font color="#444444">c        Set preconditioner (BoomerAMG = 2) and solve</font>
+<a name="line457">457</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font>, <font color="#2040a0">precond</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line458">458</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parA</font>, <font color="#2040a0">parb</font>, <font color="#2040a0">parx</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line459">459</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parA</font>, <font color="#2040a0">parb</font>, <font color="#2040a0">parx</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line460">460</a> 
+<a name="line461">461</a> <font color="#444444">c        Free memory</font>
+<a name="line462">462</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line463">463</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line464">464</a> 
+<a name="line465">465</a>       <strong>endif</strong>
+<a name="line466">466</a> 
+<a name="line467">467</a> <font color="#444444">c     Free memory</font>
+<a name="line468">468</a>       <strong>call</strong> <font color="#2040a0">HYPRE_SStructGridDestroy</font><font color="4444FF">(</font><font color="#2040a0">grid</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line469">469</a>       <strong>call</strong> <font color="#2040a0">HYPRE_SStructStencilDestroy</font><font color="4444FF">(</font><font color="#2040a0">stencil</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line470">470</a>       <strong>call</strong> <font color="#2040a0">HYPRE_SStructGraphDestroy</font><font color="4444FF">(</font><font color="#2040a0">graph</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line471">471</a>       <strong>call</strong> <font color="#2040a0">HYPRE_SStructMatrixDestroy</font><font color="4444FF">(</font><font color="#2040a0">A</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line472">472</a>       <strong>call</strong> <font color="#2040a0">HYPRE_SStructVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">b</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line473">473</a>       <strong>call</strong> <font color="#2040a0">HYPRE_SStructVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">x</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line474">474</a> 
+<a name="line475">475</a> <font color="#444444">c     Finalize MPI</font>
+<a name="line476">476</a>       <strong>call</strong> <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line477">477</a> 
+<a name="line478">478</a>       <strong>stop</strong>
+<a name="line479">479</a>       <strong>end</strong>
 
 </pre>
 <hr>
diff --git a/src/examples/README_files/ex13.c.html b/src/examples/README_files/ex13.c.html
index afbc89d..34ba65a 100644
--- a/src/examples/README_files/ex13.c.html
+++ b/src/examples/README_files/ex13.c.html
@@ -600,7 +600,7 @@
 <a name="line593">593</a>       <font color="#444444">/* Here we construct a BoomerAMG solver.  See the other SStruct examples
 <a name="line594">594</a>          as well as the Reference manual for additional solver choices. */</font>
 <a name="line595">595</a>       <font color="#2040a0">HYPRE_BoomerAMGCreate</font><font color="4444FF">(</font><font color="4444FF">&</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line596">596</a>       <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line596">596</a>       <font color="#2040a0">HYPRE_BoomerAMGSetOldDefault</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line597">597</a>       <font color="#2040a0">HYPRE_BoomerAMGSetStrongThreshold</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">0.25</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line598">598</a>       <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-6</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line599">599</a>       <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font>
diff --git a/src/examples/README_files/ex14.c.html b/src/examples/README_files/ex14.c.html
index 46e220a..4dbf4b6 100644
--- a/src/examples/README_files/ex14.c.html
+++ b/src/examples/README_files/ex14.c.html
@@ -558,7 +558,7 @@
 <a name="line551">551</a>       <font color="#444444">/* Here we construct a BoomerAMG solver.  See the other SStruct examples
 <a name="line552">552</a>          as well as the Reference manual for additional solver choices. */</font>
 <a name="line553">553</a>       <font color="#2040a0">HYPRE_BoomerAMGCreate</font><font color="4444FF">(</font><font color="4444FF">&</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line554">554</a>       <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line554">554</a>       <font color="#2040a0">HYPRE_BoomerAMGSetOldDefault</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line555">555</a>       <font color="#2040a0">HYPRE_BoomerAMGSetStrongThreshold</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">0.25</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line556">556</a>       <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-6</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line557">557</a>       <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font>
diff --git a/src/examples/README_files/ex5.c.html b/src/examples/README_files/ex5.c.html
index e675d04..65a16bc 100644
--- a/src/examples/README_files/ex5.c.html
+++ b/src/examples/README_files/ex5.c.html
@@ -316,310 +316,313 @@
 <a name="line309">309</a> 
 <a name="line310">310</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
 <a name="line311">311</a>       <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">3</font><font color="4444FF">)</font><font color="4444FF">;</font>  <font color="#444444">/* print solve info + parameters */</font>
-<a name="line312">312</a>       <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* Falgout coarsening */</font>
+<a name="line312">312</a>       <font color="#2040a0">HYPRE_BoomerAMGSetOldDefault</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* Falgout coarsening with modified classical interpolaiton */</font>
 <a name="line313">313</a>       <font color="#2040a0">HYPRE_BoomerAMGSetRelaxType</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">3</font><font color="4444FF">)</font><font color="4444FF">;</font>   <font color="#444444">/* G-S/Jacobi hybrid relaxation */</font>
-<a name="line314">314</a>       <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>   <font color="#444444">/* Sweeeps on each level */</font>
-<a name="line315">315</a>       <font color="#2040a0">HYPRE_BoomerAMGSetMaxLevels</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">20</font><font color="4444FF">)</font><font color="4444FF">;</font>  <font color="#444444">/* maximum number of levels */</font>
-<a name="line316">316</a>       <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font>      <font color="#444444">/* conv. tolerance */</font>
-<a name="line317">317</a> 
-<a name="line318">318</a>       <font color="#444444">/* Now setup and solve! */</font>
-<a name="line319">319</a>       <font color="#2040a0">HYPRE_BoomerAMGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line320">320</a>       <font color="#2040a0">HYPRE_BoomerAMGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line321">321</a> 
-<a name="line322">322</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
-<a name="line323">323</a>       <font color="#2040a0">HYPRE_BoomerAMGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line324">324</a>       <font color="#2040a0">HYPRE_BoomerAMGGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line325">325</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
-<a name="line326">326</a>       <font color="4444FF"><strong>{</strong></font>
-<a name="line327">327</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line328">328</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %d<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line329">329</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line330">330</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line331">331</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line332">332</a> 
-<a name="line333">333</a>       <font color="#444444">/* Destroy solver */</font>
-<a name="line334">334</a>       <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line335">335</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line336">336</a>    <font color="#444444">/* PCG */</font>
-<a name="line337">337</a>    <strong>else</strong> <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">50</font><font color="4444FF">)</font>
-<a name="line338">338</a>    <font color="4444FF"><strong>{</strong></font>
-<a name="line339">339</a>       <strong>int</strong> <font color="#2040a0">num_iterations</font><font color="4444FF">;</font>
-<a name="line340">340</a>       <strong>double</strong> <font color="#2040a0">final_res_norm</font><font color="4444FF">;</font>
-<a name="line341">341</a> 
-<a name="line342">342</a>       <font color="#444444">/* Create solver */</font>
-<a name="line343">343</a>       <font color="#2040a0">HYPRE_ParCSRPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line344">344</a> 
-<a name="line345">345</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
-<a name="line346">346</a>       <font color="#2040a0">HYPRE_PCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* max iterations */</font>
-<a name="line347">347</a>       <font color="#2040a0">HYPRE_PCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance */</font>
-<a name="line348">348</a>       <font color="#2040a0">HYPRE_PCGSetTwoNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* use the two norm as the stopping criteria */</font>
-<a name="line349">349</a>       <font color="#2040a0">HYPRE_PCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* prints out the iteration info */</font>
-<a name="line350">350</a>       <font color="#2040a0">HYPRE_PCGSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* needed to get run info later */</font>
-<a name="line351">351</a> 
-<a name="line352">352</a>       <font color="#444444">/* Now setup and solve! */</font>
-<a name="line353">353</a>       <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line354">354</a>       <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line355">355</a> 
-<a name="line356">356</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
-<a name="line357">357</a>       <font color="#2040a0">HYPRE_PCGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line358">358</a>       <font color="#2040a0">HYPRE_PCGGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line359">359</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
-<a name="line360">360</a>       <font color="4444FF"><strong>{</strong></font>
-<a name="line361">361</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line362">362</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %d<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line363">363</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line364">364</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line365">365</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line366">366</a> 
-<a name="line367">367</a>       <font color="#444444">/* Destroy solver */</font>
-<a name="line368">368</a>       <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line369">369</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line370">370</a>    <font color="#444444">/* PCG with AMG preconditioner */</font>
-<a name="line371">371</a>    <strong>else</strong> <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">)</font>
-<a name="line372">372</a>    <font color="4444FF"><strong>{</strong></font>
-<a name="line373">373</a>       <strong>int</strong> <font color="#2040a0">num_iterations</font><font color="4444FF">;</font>
-<a name="line374">374</a>       <strong>double</strong> <font color="#2040a0">final_res_norm</font><font color="4444FF">;</font>
-<a name="line375">375</a> 
-<a name="line376">376</a>       <font color="#444444">/* Create solver */</font>
-<a name="line377">377</a>       <font color="#2040a0">HYPRE_ParCSRPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line378">378</a> 
-<a name="line379">379</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
-<a name="line380">380</a>       <font color="#2040a0">HYPRE_PCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* max iterations */</font>
-<a name="line381">381</a>       <font color="#2040a0">HYPRE_PCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance */</font>
-<a name="line382">382</a>       <font color="#2040a0">HYPRE_PCGSetTwoNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* use the two norm as the stopping criteria */</font>
-<a name="line383">383</a>       <font color="#2040a0">HYPRE_PCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print solve info */</font>
-<a name="line384">384</a>       <font color="#2040a0">HYPRE_PCGSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* needed to get run info later */</font>
-<a name="line385">385</a> 
-<a name="line386">386</a>       <font color="#444444">/* Now set up the AMG preconditioner and specify any parameters */</font>
-<a name="line387">387</a>       <font color="#2040a0">HYPRE_BoomerAMGCreate</font><font color="4444FF">(</font><font color="4444FF">&</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line388">388</a>       <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print amg solution info */</font>
-<a name="line389">389</a>       <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line390">390</a>       <font color="#2040a0">HYPRE_BoomerAMGSetRelaxType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* Sym G.S./Jacobi hybrid */</font>
-<a name="line391">391</a>       <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line392">392</a>       <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">0.0</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance zero */</font>
-<a name="line393">393</a>       <font color="#2040a0">HYPRE_BoomerAMGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* do only one iteration! */</font>
-<a name="line394">394</a> 
-<a name="line395">395</a>       <font color="#444444">/* Set the PCG preconditioner */</font>
-<a name="line396">396</a>       <font color="#2040a0">HYPRE_PCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSolve</font>,
-<a name="line397">397</a>                           <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSetup</font>, <font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line398">398</a> 
-<a name="line399">399</a>       <font color="#444444">/* Now setup and solve! */</font>
-<a name="line400">400</a>       <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line401">401</a>       <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line402">402</a> 
-<a name="line403">403</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
-<a name="line404">404</a>       <font color="#2040a0">HYPRE_PCGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line405">405</a>       <font color="#2040a0">HYPRE_PCGGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line406">406</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
-<a name="line407">407</a>       <font color="4444FF"><strong>{</strong></font>
-<a name="line408">408</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line409">409</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %d<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line410">410</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line411">411</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line412">412</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line413">413</a> 
-<a name="line414">414</a>       <font color="#444444">/* Destroy solver and preconditioner */</font>
-<a name="line415">415</a>       <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line416">416</a>       <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line417">417</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line418">418</a>    <font color="#444444">/* PCG with Parasails Preconditioner */</font>
-<a name="line419">419</a>    <strong>else</strong> <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">8</font><font color="4444FF">)</font>
-<a name="line420">420</a>    <font color="4444FF"><strong>{</strong></font>
-<a name="line421">421</a>       <strong>int</strong>    <font color="#2040a0">num_iterations</font><font color="4444FF">;</font>
-<a name="line422">422</a>       <strong>double</strong> <font color="#2040a0">final_res_norm</font><font color="4444FF">;</font>
-<a name="line423">423</a> 
-<a name="line424">424</a>       <strong>int</strong>      <font color="#2040a0">sai_max_levels</font> <font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">;</font>
-<a name="line425">425</a>       <strong>double</strong>   <font color="#2040a0">sai_threshold</font> <font color="4444FF">=</font> <font color="#FF0000">0.1</font><font color="4444FF">;</font>
-<a name="line426">426</a>       <strong>double</strong>   <font color="#2040a0">sai_filter</font> <font color="4444FF">=</font> <font color="#FF0000">0.05</font><font color="4444FF">;</font>
-<a name="line427">427</a>       <strong>int</strong>      <font color="#2040a0">sai_sym</font> <font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">;</font>
-<a name="line428">428</a> 
-<a name="line429">429</a>       <font color="#444444">/* Create solver */</font>
-<a name="line430">430</a>       <font color="#2040a0">HYPRE_ParCSRPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line431">431</a> 
-<a name="line432">432</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
-<a name="line433">433</a>       <font color="#2040a0">HYPRE_PCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* max iterations */</font>
-<a name="line434">434</a>       <font color="#2040a0">HYPRE_PCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance */</font>
-<a name="line435">435</a>       <font color="#2040a0">HYPRE_PCGSetTwoNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* use the two norm as the stopping criteria */</font>
-<a name="line436">436</a>       <font color="#2040a0">HYPRE_PCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print solve info */</font>
-<a name="line437">437</a>       <font color="#2040a0">HYPRE_PCGSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* needed to get run info later */</font>
-<a name="line438">438</a> 
-<a name="line439">439</a>       <font color="#444444">/* Now set up the ParaSails preconditioner and specify any parameters */</font>
-<a name="line440">440</a>       <font color="#2040a0">HYPRE_ParaSailsCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line441">441</a> 
-<a name="line442">442</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
-<a name="line443">443</a>       <font color="#2040a0">HYPRE_ParaSailsSetParams</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">sai_threshold</font>, <font color="#2040a0">sai_max_levels</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line444">444</a>       <font color="#2040a0">HYPRE_ParaSailsSetFilter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">sai_filter</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line445">445</a>       <font color="#2040a0">HYPRE_ParaSailsSetSym</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">sai_sym</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line446">446</a>       <font color="#2040a0">HYPRE_ParaSailsSetLogging</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">3</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line447">447</a> 
-<a name="line448">448</a>       <font color="#444444">/* Set the PCG preconditioner */</font>
-<a name="line449">449</a>       <font color="#2040a0">HYPRE_PCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_ParaSailsSolve</font>,
-<a name="line450">450</a>                           <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_ParaSailsSetup</font>, <font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line451">451</a> 
-<a name="line452">452</a>       <font color="#444444">/* Now setup and solve! */</font>
-<a name="line453">453</a>       <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line454">454</a>       <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line455">455</a> 
-<a name="line456">456</a> 
-<a name="line457">457</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
-<a name="line458">458</a>       <font color="#2040a0">HYPRE_PCGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line459">459</a>       <font color="#2040a0">HYPRE_PCGGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line460">460</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
-<a name="line461">461</a>       <font color="4444FF"><strong>{</strong></font>
-<a name="line462">462</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line463">463</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %d<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line464">464</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line465">465</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line466">466</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line467">467</a> 
-<a name="line468">468</a>       <font color="#444444">/* Destory solver and preconditioner */</font>
-<a name="line469">469</a>       <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line470">470</a>       <font color="#2040a0">HYPRE_ParaSailsDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line471">471</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line472">472</a>    <font color="#444444">/* Flexible GMRES with  AMG Preconditioner */</font>
-<a name="line473">473</a>    <strong>else</strong> <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">61</font><font color="4444FF">)</font>
-<a name="line474">474</a>    <font color="4444FF"><strong>{</strong></font>
-<a name="line475">475</a>       <strong>int</strong>    <font color="#2040a0">num_iterations</font><font color="4444FF">;</font>
-<a name="line476">476</a>       <strong>double</strong> <font color="#2040a0">final_res_norm</font><font color="4444FF">;</font>
-<a name="line477">477</a>       <strong>int</strong>    <font color="#2040a0">restart</font> <font color="4444FF">=</font> <font color="#FF0000">30</font><font color="4444FF">;</font>
-<a name="line478">478</a>       <strong>int</strong>    <font color="#2040a0">modify</font> <font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">;</font>
-<a name="line479">479</a> 
-<a name="line480">480</a> 
-<a name="line481">481</a>       <font color="#444444">/* Create solver */</font>
-<a name="line482">482</a>       <font color="#2040a0">HYPRE_ParCSRFlexGMRESCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line483">483</a> 
-<a name="line484">484</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
-<a name="line485">485</a>       <font color="#2040a0">HYPRE_FlexGMRESSetKDim</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">restart</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line486">486</a>       <font color="#2040a0">HYPRE_FlexGMRESSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* max iterations */</font>
-<a name="line487">487</a>       <font color="#2040a0">HYPRE_FlexGMRESSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance */</font>
-<a name="line488">488</a>       <font color="#2040a0">HYPRE_FlexGMRESSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print solve info */</font>
-<a name="line489">489</a>       <font color="#2040a0">HYPRE_FlexGMRESSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* needed to get run info later */</font>
-<a name="line490">490</a> 
-<a name="line491">491</a> 
-<a name="line492">492</a>       <font color="#444444">/* Now set up the AMG preconditioner and specify any parameters */</font>
-<a name="line493">493</a>       <font color="#2040a0">HYPRE_BoomerAMGCreate</font><font color="4444FF">(</font><font color="4444FF">&</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line494">494</a>       <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print amg solution info */</font>
-<a name="line495">495</a>       <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line496">496</a>       <font color="#2040a0">HYPRE_BoomerAMGSetRelaxType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* Sym G.S./Jacobi hybrid */</font>
-<a name="line497">497</a>       <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line498">498</a>       <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">0.0</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance zero */</font>
-<a name="line499">499</a>       <font color="#2040a0">HYPRE_BoomerAMGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* do only one iteration! */</font>
-<a name="line500">500</a> 
-<a name="line501">501</a>       <font color="#444444">/* Set the FlexGMRES preconditioner */</font>
-<a name="line502">502</a>       <font color="#2040a0">HYPRE_FlexGMRESSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSolve</font>,
-<a name="line503">503</a>                           <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSetup</font>, <font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line504">504</a> 
-<a name="line505">505</a> 
-<a name="line506">506</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">modify</font><font color="4444FF">)</font>
-<a name="line507">507</a>       <font color="#444444">/* this is an optional call  - if you don't call it, hypre_FlexGMRESModifyPCDefault
-<a name="line508">508</a>          is used - which does nothing.  Otherwise, you can define your own, similar to
-<a name="line509">509</a>          the one used here */</font>
-<a name="line510">510</a>          <font color="#2040a0">HYPRE_FlexGMRESSetModifyPC</font><font color="4444FF">(</font> <font color="#2040a0">solver</font>,
-<a name="line511">511</a>                                      <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToModifyPCFcn</font><font color="4444FF">)</font> <font color="#2040a0">hypre_FlexGMRESModifyPCAMGExample</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line512">512</a> 
-<a name="line513">513</a> 
-<a name="line514">514</a>       <font color="#444444">/* Now setup and solve! */</font>
-<a name="line515">515</a>       <font color="#2040a0">HYPRE_ParCSRFlexGMRESSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line516">516</a>       <font color="#2040a0">HYPRE_ParCSRFlexGMRESSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line517">517</a> 
-<a name="line518">518</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
-<a name="line519">519</a>       <font color="#2040a0">HYPRE_FlexGMRESGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line520">520</a>       <font color="#2040a0">HYPRE_FlexGMRESGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line521">521</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
-<a name="line522">522</a>       <font color="4444FF"><strong>{</strong></font>
-<a name="line523">523</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line524">524</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %d<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line525">525</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line314">314</a>       <font color="#2040a0">HYPRE_BoomerAMGSetRelaxOrder</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>   <font color="#444444">/* uses C/F relaxation */</font>
+<a name="line315">315</a>       <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>   <font color="#444444">/* Sweeeps on each level */</font>
+<a name="line316">316</a>       <font color="#2040a0">HYPRE_BoomerAMGSetMaxLevels</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">20</font><font color="4444FF">)</font><font color="4444FF">;</font>  <font color="#444444">/* maximum number of levels */</font>
+<a name="line317">317</a>       <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font>      <font color="#444444">/* conv. tolerance */</font>
+<a name="line318">318</a> 
+<a name="line319">319</a>       <font color="#444444">/* Now setup and solve! */</font>
+<a name="line320">320</a>       <font color="#2040a0">HYPRE_BoomerAMGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line321">321</a>       <font color="#2040a0">HYPRE_BoomerAMGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line322">322</a> 
+<a name="line323">323</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
+<a name="line324">324</a>       <font color="#2040a0">HYPRE_BoomerAMGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line325">325</a>       <font color="#2040a0">HYPRE_BoomerAMGGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line326">326</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
+<a name="line327">327</a>       <font color="4444FF"><strong>{</strong></font>
+<a name="line328">328</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line329">329</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %d<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line330">330</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line331">331</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line332">332</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line333">333</a> 
+<a name="line334">334</a>       <font color="#444444">/* Destroy solver */</font>
+<a name="line335">335</a>       <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line336">336</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line337">337</a>    <font color="#444444">/* PCG */</font>
+<a name="line338">338</a>    <strong>else</strong> <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">50</font><font color="4444FF">)</font>
+<a name="line339">339</a>    <font color="4444FF"><strong>{</strong></font>
+<a name="line340">340</a>       <strong>int</strong> <font color="#2040a0">num_iterations</font><font color="4444FF">;</font>
+<a name="line341">341</a>       <strong>double</strong> <font color="#2040a0">final_res_norm</font><font color="4444FF">;</font>
+<a name="line342">342</a> 
+<a name="line343">343</a>       <font color="#444444">/* Create solver */</font>
+<a name="line344">344</a>       <font color="#2040a0">HYPRE_ParCSRPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line345">345</a> 
+<a name="line346">346</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
+<a name="line347">347</a>       <font color="#2040a0">HYPRE_PCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* max iterations */</font>
+<a name="line348">348</a>       <font color="#2040a0">HYPRE_PCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance */</font>
+<a name="line349">349</a>       <font color="#2040a0">HYPRE_PCGSetTwoNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* use the two norm as the stopping criteria */</font>
+<a name="line350">350</a>       <font color="#2040a0">HYPRE_PCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* prints out the iteration info */</font>
+<a name="line351">351</a>       <font color="#2040a0">HYPRE_PCGSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* needed to get run info later */</font>
+<a name="line352">352</a> 
+<a name="line353">353</a>       <font color="#444444">/* Now setup and solve! */</font>
+<a name="line354">354</a>       <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line355">355</a>       <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line356">356</a> 
+<a name="line357">357</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
+<a name="line358">358</a>       <font color="#2040a0">HYPRE_PCGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line359">359</a>       <font color="#2040a0">HYPRE_PCGGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line360">360</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
+<a name="line361">361</a>       <font color="4444FF"><strong>{</strong></font>
+<a name="line362">362</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line363">363</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %d<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line364">364</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line365">365</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line366">366</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line367">367</a> 
+<a name="line368">368</a>       <font color="#444444">/* Destroy solver */</font>
+<a name="line369">369</a>       <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line370">370</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line371">371</a>    <font color="#444444">/* PCG with AMG preconditioner */</font>
+<a name="line372">372</a>    <strong>else</strong> <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">)</font>
+<a name="line373">373</a>    <font color="4444FF"><strong>{</strong></font>
+<a name="line374">374</a>       <strong>int</strong> <font color="#2040a0">num_iterations</font><font color="4444FF">;</font>
+<a name="line375">375</a>       <strong>double</strong> <font color="#2040a0">final_res_norm</font><font color="4444FF">;</font>
+<a name="line376">376</a> 
+<a name="line377">377</a>       <font color="#444444">/* Create solver */</font>
+<a name="line378">378</a>       <font color="#2040a0">HYPRE_ParCSRPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line379">379</a> 
+<a name="line380">380</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
+<a name="line381">381</a>       <font color="#2040a0">HYPRE_PCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* max iterations */</font>
+<a name="line382">382</a>       <font color="#2040a0">HYPRE_PCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance */</font>
+<a name="line383">383</a>       <font color="#2040a0">HYPRE_PCGSetTwoNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* use the two norm as the stopping criteria */</font>
+<a name="line384">384</a>       <font color="#2040a0">HYPRE_PCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print solve info */</font>
+<a name="line385">385</a>       <font color="#2040a0">HYPRE_PCGSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* needed to get run info later */</font>
+<a name="line386">386</a> 
+<a name="line387">387</a>       <font color="#444444">/* Now set up the AMG preconditioner and specify any parameters */</font>
+<a name="line388">388</a>       <font color="#2040a0">HYPRE_BoomerAMGCreate</font><font color="4444FF">(</font><font color="4444FF">&</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line389">389</a>       <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print amg solution info */</font>
+<a name="line390">390</a>       <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line391">391</a>       <font color="#2040a0">HYPRE_BoomerAMGSetOldDefault</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font> 
+<a name="line392">392</a>       <font color="#2040a0">HYPRE_BoomerAMGSetRelaxType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* Sym G.S./Jacobi hybrid */</font>
+<a name="line393">393</a>       <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line394">394</a>       <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">0.0</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance zero */</font>
+<a name="line395">395</a>       <font color="#2040a0">HYPRE_BoomerAMGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* do only one iteration! */</font>
+<a name="line396">396</a> 
+<a name="line397">397</a>       <font color="#444444">/* Set the PCG preconditioner */</font>
+<a name="line398">398</a>       <font color="#2040a0">HYPRE_PCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSolve</font>,
+<a name="line399">399</a>                           <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSetup</font>, <font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line400">400</a> 
+<a name="line401">401</a>       <font color="#444444">/* Now setup and solve! */</font>
+<a name="line402">402</a>       <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line403">403</a>       <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line404">404</a> 
+<a name="line405">405</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
+<a name="line406">406</a>       <font color="#2040a0">HYPRE_PCGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line407">407</a>       <font color="#2040a0">HYPRE_PCGGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line408">408</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
+<a name="line409">409</a>       <font color="4444FF"><strong>{</strong></font>
+<a name="line410">410</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line411">411</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %d<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line412">412</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line413">413</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line414">414</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line415">415</a> 
+<a name="line416">416</a>       <font color="#444444">/* Destroy solver and preconditioner */</font>
+<a name="line417">417</a>       <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line418">418</a>       <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line419">419</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line420">420</a>    <font color="#444444">/* PCG with Parasails Preconditioner */</font>
+<a name="line421">421</a>    <strong>else</strong> <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">8</font><font color="4444FF">)</font>
+<a name="line422">422</a>    <font color="4444FF"><strong>{</strong></font>
+<a name="line423">423</a>       <strong>int</strong>    <font color="#2040a0">num_iterations</font><font color="4444FF">;</font>
+<a name="line424">424</a>       <strong>double</strong> <font color="#2040a0">final_res_norm</font><font color="4444FF">;</font>
+<a name="line425">425</a> 
+<a name="line426">426</a>       <strong>int</strong>      <font color="#2040a0">sai_max_levels</font> <font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">;</font>
+<a name="line427">427</a>       <strong>double</strong>   <font color="#2040a0">sai_threshold</font> <font color="4444FF">=</font> <font color="#FF0000">0.1</font><font color="4444FF">;</font>
+<a name="line428">428</a>       <strong>double</strong>   <font color="#2040a0">sai_filter</font> <font color="4444FF">=</font> <font color="#FF0000">0.05</font><font color="4444FF">;</font>
+<a name="line429">429</a>       <strong>int</strong>      <font color="#2040a0">sai_sym</font> <font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">;</font>
+<a name="line430">430</a> 
+<a name="line431">431</a>       <font color="#444444">/* Create solver */</font>
+<a name="line432">432</a>       <font color="#2040a0">HYPRE_ParCSRPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line433">433</a> 
+<a name="line434">434</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
+<a name="line435">435</a>       <font color="#2040a0">HYPRE_PCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* max iterations */</font>
+<a name="line436">436</a>       <font color="#2040a0">HYPRE_PCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance */</font>
+<a name="line437">437</a>       <font color="#2040a0">HYPRE_PCGSetTwoNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* use the two norm as the stopping criteria */</font>
+<a name="line438">438</a>       <font color="#2040a0">HYPRE_PCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print solve info */</font>
+<a name="line439">439</a>       <font color="#2040a0">HYPRE_PCGSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* needed to get run info later */</font>
+<a name="line440">440</a> 
+<a name="line441">441</a>       <font color="#444444">/* Now set up the ParaSails preconditioner and specify any parameters */</font>
+<a name="line442">442</a>       <font color="#2040a0">HYPRE_ParaSailsCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line443">443</a> 
+<a name="line444">444</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
+<a name="line445">445</a>       <font color="#2040a0">HYPRE_ParaSailsSetParams</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">sai_threshold</font>, <font color="#2040a0">sai_max_levels</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line446">446</a>       <font color="#2040a0">HYPRE_ParaSailsSetFilter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">sai_filter</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line447">447</a>       <font color="#2040a0">HYPRE_ParaSailsSetSym</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">sai_sym</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line448">448</a>       <font color="#2040a0">HYPRE_ParaSailsSetLogging</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">3</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line449">449</a> 
+<a name="line450">450</a>       <font color="#444444">/* Set the PCG preconditioner */</font>
+<a name="line451">451</a>       <font color="#2040a0">HYPRE_PCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_ParaSailsSolve</font>,
+<a name="line452">452</a>                           <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_ParaSailsSetup</font>, <font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line453">453</a> 
+<a name="line454">454</a>       <font color="#444444">/* Now setup and solve! */</font>
+<a name="line455">455</a>       <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line456">456</a>       <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line457">457</a> 
+<a name="line458">458</a> 
+<a name="line459">459</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
+<a name="line460">460</a>       <font color="#2040a0">HYPRE_PCGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line461">461</a>       <font color="#2040a0">HYPRE_PCGGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line462">462</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
+<a name="line463">463</a>       <font color="4444FF"><strong>{</strong></font>
+<a name="line464">464</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line465">465</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %d<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line466">466</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line467">467</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line468">468</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line469">469</a> 
+<a name="line470">470</a>       <font color="#444444">/* Destory solver and preconditioner */</font>
+<a name="line471">471</a>       <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line472">472</a>       <font color="#2040a0">HYPRE_ParaSailsDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line473">473</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line474">474</a>    <font color="#444444">/* Flexible GMRES with  AMG Preconditioner */</font>
+<a name="line475">475</a>    <strong>else</strong> <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">61</font><font color="4444FF">)</font>
+<a name="line476">476</a>    <font color="4444FF"><strong>{</strong></font>
+<a name="line477">477</a>       <strong>int</strong>    <font color="#2040a0">num_iterations</font><font color="4444FF">;</font>
+<a name="line478">478</a>       <strong>double</strong> <font color="#2040a0">final_res_norm</font><font color="4444FF">;</font>
+<a name="line479">479</a>       <strong>int</strong>    <font color="#2040a0">restart</font> <font color="4444FF">=</font> <font color="#FF0000">30</font><font color="4444FF">;</font>
+<a name="line480">480</a>       <strong>int</strong>    <font color="#2040a0">modify</font> <font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">;</font>
+<a name="line481">481</a> 
+<a name="line482">482</a> 
+<a name="line483">483</a>       <font color="#444444">/* Create solver */</font>
+<a name="line484">484</a>       <font color="#2040a0">HYPRE_ParCSRFlexGMRESCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line485">485</a> 
+<a name="line486">486</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
+<a name="line487">487</a>       <font color="#2040a0">HYPRE_FlexGMRESSetKDim</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">restart</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line488">488</a>       <font color="#2040a0">HYPRE_FlexGMRESSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* max iterations */</font>
+<a name="line489">489</a>       <font color="#2040a0">HYPRE_FlexGMRESSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance */</font>
+<a name="line490">490</a>       <font color="#2040a0">HYPRE_FlexGMRESSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print solve info */</font>
+<a name="line491">491</a>       <font color="#2040a0">HYPRE_FlexGMRESSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* needed to get run info later */</font>
+<a name="line492">492</a> 
+<a name="line493">493</a> 
+<a name="line494">494</a>       <font color="#444444">/* Now set up the AMG preconditioner and specify any parameters */</font>
+<a name="line495">495</a>       <font color="#2040a0">HYPRE_BoomerAMGCreate</font><font color="4444FF">(</font><font color="4444FF">&</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line496">496</a>       <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print amg solution info */</font>
+<a name="line497">497</a>       <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line498">498</a>       <font color="#2040a0">HYPRE_BoomerAMGSetOldDefault</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line499">499</a>       <font color="#2040a0">HYPRE_BoomerAMGSetRelaxType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* Sym G.S./Jacobi hybrid */</font>
+<a name="line500">500</a>       <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line501">501</a>       <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">0.0</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance zero */</font>
+<a name="line502">502</a>       <font color="#2040a0">HYPRE_BoomerAMGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* do only one iteration! */</font>
+<a name="line503">503</a> 
+<a name="line504">504</a>       <font color="#444444">/* Set the FlexGMRES preconditioner */</font>
+<a name="line505">505</a>       <font color="#2040a0">HYPRE_FlexGMRESSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSolve</font>,
+<a name="line506">506</a>                           <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSetup</font>, <font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line507">507</a> 
+<a name="line508">508</a> 
+<a name="line509">509</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">modify</font><font color="4444FF">)</font>
+<a name="line510">510</a>       <font color="#444444">/* this is an optional call  - if you don't call it, hypre_FlexGMRESModifyPCDefault
+<a name="line511">511</a>          is used - which does nothing.  Otherwise, you can define your own, similar to
+<a name="line512">512</a>          the one used here */</font>
+<a name="line513">513</a>          <font color="#2040a0">HYPRE_FlexGMRESSetModifyPC</font><font color="4444FF">(</font> <font color="#2040a0">solver</font>,
+<a name="line514">514</a>                                      <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToModifyPCFcn</font><font color="4444FF">)</font> <font color="#2040a0">hypre_FlexGMRESModifyPCAMGExample</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line515">515</a> 
+<a name="line516">516</a> 
+<a name="line517">517</a>       <font color="#444444">/* Now setup and solve! */</font>
+<a name="line518">518</a>       <font color="#2040a0">HYPRE_ParCSRFlexGMRESSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line519">519</a>       <font color="#2040a0">HYPRE_ParCSRFlexGMRESSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line520">520</a> 
+<a name="line521">521</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
+<a name="line522">522</a>       <font color="#2040a0">HYPRE_FlexGMRESGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line523">523</a>       <font color="#2040a0">HYPRE_FlexGMRESGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line524">524</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
+<a name="line525">525</a>       <font color="4444FF"><strong>{</strong></font>
 <a name="line526">526</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line527">527</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line528">528</a> 
-<a name="line529">529</a>       <font color="#444444">/* Destory solver and preconditioner */</font>
-<a name="line530">530</a>       <font color="#2040a0">HYPRE_ParCSRFlexGMRESDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line531">531</a>       <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line532">532</a> 
-<a name="line533">533</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line534">534</a>    <strong>else</strong>
-<a name="line535">535</a>    <font color="4444FF"><strong>{</strong></font>
-<a name="line536">536</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font><font color="#FF0000">0</font><font color="4444FF">)</font> <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Invalid solver id specified.<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line537">537</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line538">538</a> 
-<a name="line539">539</a>    <font color="#444444">/* Save the solution for GLVis visualization, see vis/glvis-ex5.sh */</font>
-<a name="line540">540</a>    <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">vis</font><font color="4444FF">)</font>
-<a name="line541">541</a>    <font color="4444FF"><strong>{</strong></font>
-<a name="line542">542</a>       <font color="#2040a0">FILE</font> <font color="4444FF">*</font><font color="#2040a0">file</font><font color="4444FF">;</font>
-<a name="line543">543</a>       <strong>char</strong> <font color="#2040a0">filename</font><font color="4444FF">[</font><font color="#FF0000">255</font><font color="4444FF">]</font><font color="4444FF">;</font>
-<a name="line544">544</a> 
-<a name="line545">545</a>       <strong>int</strong> <font color="#2040a0">nvalues</font> <font color="4444FF">=</font> <font color="#2040a0">local_size</font><font color="4444FF">;</font>
-<a name="line546">546</a>       <strong>int</strong> <font color="4444FF">*</font><font color="#2040a0">rows</font> <font color="4444FF">=</font> <font color="4444FF">(</font><strong>int</strong><font color="4444FF">*</font><font color="4444FF">)</font> <font color="#2040a0">calloc</font><font color="4444FF">(</font><font color="#2040a0">nvalues</font>, <strong>sizeof</strong><font color="4444FF">(</font><strong>int</strong><font color="4444FF">)</font><font color="4444FF">)</font><font  [...]
-<a name="line547">547</a>       <strong>double</strong> <font color="4444FF">*</font><font color="#2040a0">values</font> <font color="4444FF">=</font>  <font color="4444FF">(</font><strong>double</strong><font color="4444FF">*</font><font color="4444FF">)</font> <font color="#2040a0">calloc</font><font color="4444FF">(</font><font color="#2040a0">nvalues</font>, <strong>sizeof</strong><font color="4444FF">(</font><strong>double</strong><font color="4444FF">)</font><font color="4444FF">)< [...]
-<a name="line548">548</a> 
-<a name="line549">549</a>       <strong>for</strong> <font color="4444FF">(</font><font color="#2040a0">i</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font> <font color="#2040a0">i</font> <font color="4444FF"><</font> <font color="#2040a0">nvalues</font><font color="4444FF">;</font> <font color="#2040a0">i</font><font color="4444FF">+</font><font color="4444FF">+</font><font color="4444FF">)</font>
-<a name="line550">550</a>          <font color="#2040a0">rows</font><font color="4444FF">[</font><font color="#2040a0">i</font><font color="4444FF">]</font> <font color="4444FF">=</font> <font color="#2040a0">ilower</font> <font color="4444FF">+</font> <font color="#2040a0">i</font><font color="4444FF">;</font>
+<a name="line527">527</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %d<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line528">528</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line529">529</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line530">530</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line531">531</a> 
+<a name="line532">532</a>       <font color="#444444">/* Destory solver and preconditioner */</font>
+<a name="line533">533</a>       <font color="#2040a0">HYPRE_ParCSRFlexGMRESDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line534">534</a>       <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line535">535</a> 
+<a name="line536">536</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line537">537</a>    <strong>else</strong>
+<a name="line538">538</a>    <font color="4444FF"><strong>{</strong></font>
+<a name="line539">539</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font><font color="#FF0000">0</font><font color="4444FF">)</font> <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Invalid solver id specified.<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line540">540</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line541">541</a> 
+<a name="line542">542</a>    <font color="#444444">/* Save the solution for GLVis visualization, see vis/glvis-ex5.sh */</font>
+<a name="line543">543</a>    <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">vis</font><font color="4444FF">)</font>
+<a name="line544">544</a>    <font color="4444FF"><strong>{</strong></font>
+<a name="line545">545</a>       <font color="#2040a0">FILE</font> <font color="4444FF">*</font><font color="#2040a0">file</font><font color="4444FF">;</font>
+<a name="line546">546</a>       <strong>char</strong> <font color="#2040a0">filename</font><font color="4444FF">[</font><font color="#FF0000">255</font><font color="4444FF">]</font><font color="4444FF">;</font>
+<a name="line547">547</a> 
+<a name="line548">548</a>       <strong>int</strong> <font color="#2040a0">nvalues</font> <font color="4444FF">=</font> <font color="#2040a0">local_size</font><font color="4444FF">;</font>
+<a name="line549">549</a>       <strong>int</strong> <font color="4444FF">*</font><font color="#2040a0">rows</font> <font color="4444FF">=</font> <font color="4444FF">(</font><strong>int</strong><font color="4444FF">*</font><font color="4444FF">)</font> <font color="#2040a0">calloc</font><font color="4444FF">(</font><font color="#2040a0">nvalues</font>, <strong>sizeof</strong><font color="4444FF">(</font><strong>int</strong><font color="4444FF">)</font><font color="4444FF">)</font><font  [...]
+<a name="line550">550</a>       <strong>double</strong> <font color="4444FF">*</font><font color="#2040a0">values</font> <font color="4444FF">=</font>  <font color="4444FF">(</font><strong>double</strong><font color="4444FF">*</font><font color="4444FF">)</font> <font color="#2040a0">calloc</font><font color="4444FF">(</font><font color="#2040a0">nvalues</font>, <strong>sizeof</strong><font color="4444FF">(</font><strong>double</strong><font color="4444FF">)</font><font color="4444FF">)< [...]
 <a name="line551">551</a> 
-<a name="line552">552</a>       <font color="#444444">/* get the local solution */</font>
-<a name="line553">553</a>       <font color="#2040a0">HYPRE_IJVectorGetValues</font><font color="4444FF">(</font><font color="#2040a0">x</font>, <font color="#2040a0">nvalues</font>, <font color="#2040a0">rows</font>, <font color="#2040a0">values</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line552">552</a>       <strong>for</strong> <font color="4444FF">(</font><font color="#2040a0">i</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font> <font color="#2040a0">i</font> <font color="4444FF"><</font> <font color="#2040a0">nvalues</font><font color="4444FF">;</font> <font color="#2040a0">i</font><font color="4444FF">+</font><font color="4444FF">+</font><font color="4444FF">)</font>
+<a name="line553">553</a>          <font color="#2040a0">rows</font><font color="4444FF">[</font><font color="#2040a0">i</font><font color="4444FF">]</font> <font color="4444FF">=</font> <font color="#2040a0">ilower</font> <font color="4444FF">+</font> <font color="#2040a0">i</font><font color="4444FF">;</font>
 <a name="line554">554</a> 
-<a name="line555">555</a>       <font color="#2040a0">sprintf</font><font color="4444FF">(</font><font color="#2040a0">filename</font>, <font color="#008000">"%s.%06d"</font>, <font color="#008000">"vis/ex5.sol"</font>, <font color="#2040a0">myid</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line556">556</a>       <strong>if</strong> <font color="4444FF">(</font><font color="4444FF">(</font><font color="#2040a0">file</font> <font color="4444FF">=</font> <font color="#2040a0">fopen</font><font color="4444FF">(</font><font color="#2040a0">filename</font>, <font color="#008000">"w"</font><font color="4444FF">)</font><font color="4444FF">)</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#2040a0">NULL</font><font color="4444FF">)</font>
-<a name="line557">557</a>       <font color="4444FF"><strong>{</strong></font>
-<a name="line558">558</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Error: can't open output file %s<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">filename</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line559">559</a>          <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line560">560</a>          <font color="#2040a0">exit</font><font color="4444FF">(</font><font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line561">561</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line562">562</a> 
-<a name="line563">563</a>       <font color="#444444">/* save solution */</font>
-<a name="line564">564</a>       <strong>for</strong> <font color="4444FF">(</font><font color="#2040a0">i</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font> <font color="#2040a0">i</font> <font color="4444FF"><</font> <font color="#2040a0">nvalues</font><font color="4444FF">;</font> <font color="#2040a0">i</font><font color="4444FF">+</font><font color="4444FF">+</font><font color="4444FF">)</font>
-<a name="line565">565</a>          <font color="#2040a0">fprintf</font><font color="4444FF">(</font><font color="#2040a0">file</font>, <font color="#008000">"%.14e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">values</font><font color="4444FF">[</font><font color="#2040a0">i</font><font color="4444FF">]</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line566">566</a> 
-<a name="line567">567</a>       <font color="#2040a0">fflush</font><font color="4444FF">(</font><font color="#2040a0">file</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line568">568</a>       <font color="#2040a0">fclose</font><font color="4444FF">(</font><font color="#2040a0">file</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line555">555</a>       <font color="#444444">/* get the local solution */</font>
+<a name="line556">556</a>       <font color="#2040a0">HYPRE_IJVectorGetValues</font><font color="4444FF">(</font><font color="#2040a0">x</font>, <font color="#2040a0">nvalues</font>, <font color="#2040a0">rows</font>, <font color="#2040a0">values</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line557">557</a> 
+<a name="line558">558</a>       <font color="#2040a0">sprintf</font><font color="4444FF">(</font><font color="#2040a0">filename</font>, <font color="#008000">"%s.%06d"</font>, <font color="#008000">"vis/ex5.sol"</font>, <font color="#2040a0">myid</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line559">559</a>       <strong>if</strong> <font color="4444FF">(</font><font color="4444FF">(</font><font color="#2040a0">file</font> <font color="4444FF">=</font> <font color="#2040a0">fopen</font><font color="4444FF">(</font><font color="#2040a0">filename</font>, <font color="#008000">"w"</font><font color="4444FF">)</font><font color="4444FF">)</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#2040a0">NULL</font><font color="4444FF">)</font>
+<a name="line560">560</a>       <font color="4444FF"><strong>{</strong></font>
+<a name="line561">561</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Error: can't open output file %s<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">filename</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line562">562</a>          <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line563">563</a>          <font color="#2040a0">exit</font><font color="4444FF">(</font><font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line564">564</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line565">565</a> 
+<a name="line566">566</a>       <font color="#444444">/* save solution */</font>
+<a name="line567">567</a>       <strong>for</strong> <font color="4444FF">(</font><font color="#2040a0">i</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font> <font color="#2040a0">i</font> <font color="4444FF"><</font> <font color="#2040a0">nvalues</font><font color="4444FF">;</font> <font color="#2040a0">i</font><font color="4444FF">+</font><font color="4444FF">+</font><font color="4444FF">)</font>
+<a name="line568">568</a>          <font color="#2040a0">fprintf</font><font color="4444FF">(</font><font color="#2040a0">file</font>, <font color="#008000">"%.14e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">values</font><font color="4444FF">[</font><font color="#2040a0">i</font><font color="4444FF">]</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line569">569</a> 
-<a name="line570">570</a>       <font color="#2040a0">free</font><font color="4444FF">(</font><font color="#2040a0">rows</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line571">571</a>       <font color="#2040a0">free</font><font color="4444FF">(</font><font color="#2040a0">values</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line570">570</a>       <font color="#2040a0">fflush</font><font color="4444FF">(</font><font color="#2040a0">file</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line571">571</a>       <font color="#2040a0">fclose</font><font color="4444FF">(</font><font color="#2040a0">file</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line572">572</a> 
-<a name="line573">573</a>       <font color="#444444">/* save global finite element mesh */</font>
-<a name="line574">574</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
-<a name="line575">575</a>          <font color="#2040a0">GLVis_PrintGlobalSquareMesh</font><font color="4444FF">(</font><font color="#008000">"vis/ex5.mesh"</font>, <font color="#2040a0">n</font><font color="4444FF">-</font><font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line576">576</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line577">577</a> 
-<a name="line578">578</a>    <font color="#444444">/* Clean up */</font>
-<a name="line579">579</a>    <font color="#2040a0">HYPRE_IJMatrixDestroy</font><font color="4444FF">(</font><font color="#2040a0">A</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line580">580</a>    <font color="#2040a0">HYPRE_IJVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">b</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line581">581</a>    <font color="#2040a0">HYPRE_IJVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line582">582</a> 
-<a name="line583">583</a>    <font color="#444444">/* Finalize MPI*/</font>
-<a name="line584">584</a>    <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line573">573</a>       <font color="#2040a0">free</font><font color="4444FF">(</font><font color="#2040a0">rows</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line574">574</a>       <font color="#2040a0">free</font><font color="4444FF">(</font><font color="#2040a0">values</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line575">575</a> 
+<a name="line576">576</a>       <font color="#444444">/* save global finite element mesh */</font>
+<a name="line577">577</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
+<a name="line578">578</a>          <font color="#2040a0">GLVis_PrintGlobalSquareMesh</font><font color="4444FF">(</font><font color="#008000">"vis/ex5.mesh"</font>, <font color="#2040a0">n</font><font color="4444FF">-</font><font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line579">579</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line580">580</a> 
+<a name="line581">581</a>    <font color="#444444">/* Clean up */</font>
+<a name="line582">582</a>    <font color="#2040a0">HYPRE_IJMatrixDestroy</font><font color="4444FF">(</font><font color="#2040a0">A</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line583">583</a>    <font color="#2040a0">HYPRE_IJVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">b</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line584">584</a>    <font color="#2040a0">HYPRE_IJVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">x</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line585">585</a> 
-<a name="line586">586</a>    <strong>return</strong><font color="4444FF">(</font><font color="#FF0000">0</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line587">587</a> <font color="4444FF"><strong>}</strong></font>
+<a name="line586">586</a>    <font color="#444444">/* Finalize MPI*/</font>
+<a name="line587">587</a>    <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line588">588</a> 
-<a name="line589">589</a> <font color="#444444">/*--------------------------------------------------------------------------
-<a name="line590">590</a>    hypre_FlexGMRESModifyPCAMGExample -
+<a name="line589">589</a>    <strong>return</strong><font color="4444FF">(</font><font color="#FF0000">0</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line590">590</a> <font color="4444FF"><strong>}</strong></font>
 <a name="line591">591</a> 
-<a name="line592">592</a>     This is an example (not recommended)
-<a name="line593">593</a>    of how we can modify things about AMG that
-<a name="line594">594</a>    affect the solve phase based on how FlexGMRES is doing...For
-<a name="line595">595</a>    another preconditioner it may make sense to modify the tolerance..
-<a name="line596">596</a> 
-<a name="line597">597</a>  *--------------------------------------------------------------------------*/</font>
-<a name="line598">598</a> 
-<a name="line599">599</a> <strong>int</strong> <font color="#2040a0">hypre_FlexGMRESModifyPCAMGExample</font><font color="4444FF">(</font><strong>void</strong> <font color="4444FF">*</font><font color="#2040a0">precond_data</font>, <strong>int</strong> <font color="#2040a0">iterations</font>,
-<a name="line600">600</a>                                    <strong>double</strong> <font color="#2040a0">rel_residual_norm</font><font color="4444FF">)</font>
-<a name="line601">601</a> <font color="4444FF"><strong>{</strong></font>
-<a name="line602">602</a> 
-<a name="line603">603</a> 
-<a name="line604">604</a>    <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">rel_residual_norm</font> <font color="4444FF">></font> .<font color="#FF0000">1</font><font color="4444FF">)</font>
-<a name="line605">605</a>    <font color="4444FF"><strong>{</strong></font>
-<a name="line606">606</a> 	   <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="4444FF">(</font><font color="#2040a0">HYPRE_Solver</font><font color="4444FF">)</font><font color="#2040a0">precond_data</font>, <font color="#FF0000">10</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line607">607</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line608">608</a>    <strong>else</strong>
-<a name="line609">609</a>    <font color="4444FF"><strong>{</strong></font>
-<a name="line610">610</a>       <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="4444FF">(</font><font color="#2040a0">HYPRE_Solver</font><font color="4444FF">)</font><font color="#2040a0">precond_data</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line611">611</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line612">612</a> 
-<a name="line613">613</a> 
-<a name="line614">614</a>    <strong>return</strong> <font color="#FF0000">0</font><font color="4444FF">;</font>
-<a name="line615">615</a> <font color="4444FF"><strong>}</strong></font>
+<a name="line592">592</a> <font color="#444444">/*--------------------------------------------------------------------------
+<a name="line593">593</a>    hypre_FlexGMRESModifyPCAMGExample -
+<a name="line594">594</a> 
+<a name="line595">595</a>     This is an example (not recommended)
+<a name="line596">596</a>    of how we can modify things about AMG that
+<a name="line597">597</a>    affect the solve phase based on how FlexGMRES is doing...For
+<a name="line598">598</a>    another preconditioner it may make sense to modify the tolerance..
+<a name="line599">599</a> 
+<a name="line600">600</a>  *--------------------------------------------------------------------------*/</font>
+<a name="line601">601</a> 
+<a name="line602">602</a> <strong>int</strong> <font color="#2040a0">hypre_FlexGMRESModifyPCAMGExample</font><font color="4444FF">(</font><strong>void</strong> <font color="4444FF">*</font><font color="#2040a0">precond_data</font>, <strong>int</strong> <font color="#2040a0">iterations</font>,
+<a name="line603">603</a>                                    <strong>double</strong> <font color="#2040a0">rel_residual_norm</font><font color="4444FF">)</font>
+<a name="line604">604</a> <font color="4444FF"><strong>{</strong></font>
+<a name="line605">605</a> 
+<a name="line606">606</a> 
+<a name="line607">607</a>    <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">rel_residual_norm</font> <font color="4444FF">></font> .<font color="#FF0000">1</font><font color="4444FF">)</font>
+<a name="line608">608</a>    <font color="4444FF"><strong>{</strong></font>
+<a name="line609">609</a> 	   <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="4444FF">(</font><font color="#2040a0">HYPRE_Solver</font><font color="4444FF">)</font><font color="#2040a0">precond_data</font>, <font color="#FF0000">10</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line610">610</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line611">611</a>    <strong>else</strong>
+<a name="line612">612</a>    <font color="4444FF"><strong>{</strong></font>
+<a name="line613">613</a>       <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="4444FF">(</font><font color="#2040a0">HYPRE_Solver</font><font color="4444FF">)</font><font color="#2040a0">precond_data</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line614">614</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line615">615</a> 
+<a name="line616">616</a> 
+<a name="line617">617</a>    <strong>return</strong> <font color="#FF0000">0</font><font color="4444FF">;</font>
+<a name="line618">618</a> <font color="4444FF"><strong>}</strong></font>
 
 </pre>
 <hr>
diff --git a/src/examples/README_files/ex5big.c.html b/src/examples/README_files/ex5big.c.html
index 950709a..b5fad5c 100644
--- a/src/examples/README_files/ex5big.c.html
+++ b/src/examples/README_files/ex5big.c.html
@@ -315,271 +315,274 @@
 <a name="line308">308</a> 
 <a name="line309">309</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
 <a name="line310">310</a>       <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">3</font><font color="4444FF">)</font><font color="4444FF">;</font>  <font color="#444444">/* print solve info + parameters */</font>
-<a name="line311">311</a>       <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* Falgout coarsening */</font>
+<a name="line311">311</a>       <font color="#2040a0">HYPRE_BoomerAMGSetOldDefault</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* Falgout coarsening with modified classical interpolation */</font>
 <a name="line312">312</a>       <font color="#2040a0">HYPRE_BoomerAMGSetRelaxType</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">3</font><font color="4444FF">)</font><font color="4444FF">;</font>   <font color="#444444">/* G-S/Jacobi hybrid relaxation */</font>
-<a name="line313">313</a>       <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>   <font color="#444444">/* Sweeeps on each level */</font>
-<a name="line314">314</a>       <font color="#2040a0">HYPRE_BoomerAMGSetMaxLevels</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">20</font><font color="4444FF">)</font><font color="4444FF">;</font>  <font color="#444444">/* maximum number of levels */</font>
-<a name="line315">315</a>       <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font>      <font color="#444444">/* conv. tolerance */</font>
-<a name="line316">316</a> 
-<a name="line317">317</a>       <font color="#444444">/* Now setup and solve! */</font>
-<a name="line318">318</a>       <font color="#2040a0">HYPRE_BoomerAMGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line319">319</a>       <font color="#2040a0">HYPRE_BoomerAMGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line320">320</a> 
-<a name="line321">321</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
-<a name="line322">322</a>       <font color="#2040a0">HYPRE_BoomerAMGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line323">323</a>       <font color="#2040a0">HYPRE_BoomerAMGGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line324">324</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
-<a name="line325">325</a>       <font color="4444FF"><strong>{</strong></font>
-<a name="line326">326</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line327">327</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %lld<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line328">328</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line329">329</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line330">330</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line331">331</a> 
-<a name="line332">332</a>       <font color="#444444">/* Destroy solver */</font>
-<a name="line333">333</a>       <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line334">334</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line335">335</a>    <font color="#444444">/* PCG */</font>
-<a name="line336">336</a>    <strong>else</strong> <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">50</font><font color="4444FF">)</font>
-<a name="line337">337</a>    <font color="4444FF"><strong>{</strong></font>
-<a name="line338">338</a>       <font color="#2040a0">HYPRE_Int</font> <font color="#2040a0">num_iterations</font><font color="4444FF">;</font>
-<a name="line339">339</a>       <strong>double</strong> <font color="#2040a0">final_res_norm</font><font color="4444FF">;</font>
-<a name="line340">340</a> 
-<a name="line341">341</a>       <font color="#444444">/* Create solver */</font>
-<a name="line342">342</a>       <font color="#2040a0">HYPRE_ParCSRPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line343">343</a> 
-<a name="line344">344</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
-<a name="line345">345</a>       <font color="#2040a0">HYPRE_PCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* max iterations */</font>
-<a name="line346">346</a>       <font color="#2040a0">HYPRE_PCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance */</font>
-<a name="line347">347</a>       <font color="#2040a0">HYPRE_PCGSetTwoNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* use the two norm as the stopping criteria */</font>
-<a name="line348">348</a>       <font color="#2040a0">HYPRE_PCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* prints out the iteration info */</font>
-<a name="line349">349</a>       <font color="#2040a0">HYPRE_PCGSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* needed to get run info later */</font>
-<a name="line350">350</a> 
-<a name="line351">351</a>       <font color="#444444">/* Now setup and solve! */</font>
-<a name="line352">352</a>       <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line353">353</a>       <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line354">354</a> 
-<a name="line355">355</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
-<a name="line356">356</a>       <font color="#2040a0">HYPRE_PCGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line357">357</a>       <font color="#2040a0">HYPRE_PCGGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line358">358</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
-<a name="line359">359</a>       <font color="4444FF"><strong>{</strong></font>
-<a name="line360">360</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line361">361</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %lld<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line362">362</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line363">363</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line364">364</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line365">365</a> 
-<a name="line366">366</a>       <font color="#444444">/* Destroy solver */</font>
-<a name="line367">367</a>       <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line368">368</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line369">369</a>    <font color="#444444">/* PCG with AMG preconditioner */</font>
-<a name="line370">370</a>    <strong>else</strong> <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">)</font>
-<a name="line371">371</a>    <font color="4444FF"><strong>{</strong></font>
-<a name="line372">372</a>       <font color="#2040a0">HYPRE_Int</font> <font color="#2040a0">num_iterations</font><font color="4444FF">;</font>
-<a name="line373">373</a>       <strong>double</strong> <font color="#2040a0">final_res_norm</font><font color="4444FF">;</font>
-<a name="line374">374</a> 
-<a name="line375">375</a>       <font color="#444444">/* Create solver */</font>
-<a name="line376">376</a>       <font color="#2040a0">HYPRE_ParCSRPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line377">377</a> 
-<a name="line378">378</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
-<a name="line379">379</a>       <font color="#2040a0">HYPRE_PCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* max iterations */</font>
-<a name="line380">380</a>       <font color="#2040a0">HYPRE_PCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance */</font>
-<a name="line381">381</a>       <font color="#2040a0">HYPRE_PCGSetTwoNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* use the two norm as the stopping criteria */</font>
-<a name="line382">382</a>       <font color="#2040a0">HYPRE_PCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print solve info */</font>
-<a name="line383">383</a>       <font color="#2040a0">HYPRE_PCGSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* needed to get run info later */</font>
-<a name="line384">384</a> 
-<a name="line385">385</a>       <font color="#444444">/* Now set up the AMG preconditioner and specify any parameters */</font>
-<a name="line386">386</a>       <font color="#2040a0">HYPRE_BoomerAMGCreate</font><font color="4444FF">(</font><font color="4444FF">&</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line387">387</a>       <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print amg solution info */</font>
-<a name="line388">388</a>       <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line389">389</a>       <font color="#2040a0">HYPRE_BoomerAMGSetRelaxType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* Sym G.S./Jacobi hybrid */</font>
-<a name="line390">390</a>       <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line391">391</a>       <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">0.0</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance zero */</font>
-<a name="line392">392</a>       <font color="#2040a0">HYPRE_BoomerAMGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* do only one iteration! */</font>
-<a name="line393">393</a> 
-<a name="line394">394</a>       <font color="#444444">/* Set the PCG preconditioner */</font>
-<a name="line395">395</a>       <font color="#2040a0">HYPRE_PCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSolve</font>,
-<a name="line396">396</a>                           <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSetup</font>, <font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line397">397</a> 
-<a name="line398">398</a>       <font color="#444444">/* Now setup and solve! */</font>
-<a name="line399">399</a>       <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line400">400</a>       <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line401">401</a> 
-<a name="line402">402</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
-<a name="line403">403</a>       <font color="#2040a0">HYPRE_PCGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line404">404</a>       <font color="#2040a0">HYPRE_PCGGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line405">405</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
-<a name="line406">406</a>       <font color="4444FF"><strong>{</strong></font>
-<a name="line407">407</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line408">408</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %lld<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line409">409</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line410">410</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line411">411</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line412">412</a> 
-<a name="line413">413</a>       <font color="#444444">/* Destroy solver and preconditioner */</font>
-<a name="line414">414</a>       <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line415">415</a>       <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line416">416</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line417">417</a>    <font color="#444444">/* PCG with Parasails Preconditioner */</font>
-<a name="line418">418</a>    <strong>else</strong> <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">8</font><font color="4444FF">)</font>
-<a name="line419">419</a>    <font color="4444FF"><strong>{</strong></font>
-<a name="line420">420</a>       <font color="#2040a0">HYPRE_Int</font> <font color="#2040a0">num_iterations</font><font color="4444FF">;</font>
-<a name="line421">421</a>       <strong>double</strong> <font color="#2040a0">final_res_norm</font><font color="4444FF">;</font>
-<a name="line422">422</a> 
-<a name="line423">423</a>       <strong>int</strong>      <font color="#2040a0">sai_max_levels</font> <font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">;</font>
-<a name="line424">424</a>       <strong>double</strong>   <font color="#2040a0">sai_threshold</font> <font color="4444FF">=</font> <font color="#FF0000">0.1</font><font color="4444FF">;</font>
-<a name="line425">425</a>       <strong>double</strong>   <font color="#2040a0">sai_filter</font> <font color="4444FF">=</font> <font color="#FF0000">0.05</font><font color="4444FF">;</font>
-<a name="line426">426</a>       <strong>int</strong>      <font color="#2040a0">sai_sym</font> <font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">;</font>
-<a name="line427">427</a> 
-<a name="line428">428</a>       <font color="#444444">/* Create solver */</font>
-<a name="line429">429</a>       <font color="#2040a0">HYPRE_ParCSRPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line430">430</a> 
-<a name="line431">431</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
-<a name="line432">432</a>       <font color="#2040a0">HYPRE_PCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* max iterations */</font>
-<a name="line433">433</a>       <font color="#2040a0">HYPRE_PCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance */</font>
-<a name="line434">434</a>       <font color="#2040a0">HYPRE_PCGSetTwoNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* use the two norm as the stopping criteria */</font>
-<a name="line435">435</a>       <font color="#2040a0">HYPRE_PCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print solve info */</font>
-<a name="line436">436</a>       <font color="#2040a0">HYPRE_PCGSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* needed to get run info later */</font>
-<a name="line437">437</a> 
-<a name="line438">438</a>       <font color="#444444">/* Now set up the ParaSails preconditioner and specify any parameters */</font>
-<a name="line439">439</a>       <font color="#2040a0">HYPRE_ParaSailsCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line440">440</a> 
-<a name="line441">441</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
-<a name="line442">442</a>       <font color="#2040a0">HYPRE_ParaSailsSetParams</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">sai_threshold</font>, <font color="#2040a0">sai_max_levels</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line443">443</a>       <font color="#2040a0">HYPRE_ParaSailsSetFilter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">sai_filter</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line444">444</a>       <font color="#2040a0">HYPRE_ParaSailsSetSym</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">sai_sym</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line445">445</a>       <font color="#2040a0">HYPRE_ParaSailsSetLogging</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">3</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line446">446</a> 
-<a name="line447">447</a>       <font color="#444444">/* Set the PCG preconditioner */</font>
-<a name="line448">448</a>       <font color="#2040a0">HYPRE_PCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_ParaSailsSolve</font>,
-<a name="line449">449</a>                           <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_ParaSailsSetup</font>, <font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line450">450</a> 
-<a name="line451">451</a>       <font color="#444444">/* Now setup and solve! */</font>
-<a name="line452">452</a>       <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line453">453</a>       <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line454">454</a> 
-<a name="line455">455</a> 
-<a name="line456">456</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
-<a name="line457">457</a>       <font color="#2040a0">HYPRE_PCGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line458">458</a>       <font color="#2040a0">HYPRE_PCGGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line459">459</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
-<a name="line460">460</a>       <font color="4444FF"><strong>{</strong></font>
-<a name="line461">461</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line462">462</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %lld<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line463">463</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line464">464</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line465">465</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line466">466</a> 
-<a name="line467">467</a>       <font color="#444444">/* Destory solver and preconditioner */</font>
-<a name="line468">468</a>       <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line469">469</a>       <font color="#2040a0">HYPRE_ParaSailsDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line470">470</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line471">471</a>    <font color="#444444">/* Flexible GMRES with  AMG Preconditioner */</font>
-<a name="line472">472</a>    <strong>else</strong> <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">61</font><font color="4444FF">)</font>
-<a name="line473">473</a>    <font color="4444FF"><strong>{</strong></font>
-<a name="line474">474</a>       <font color="#2040a0">HYPRE_Int</font> <font color="#2040a0">num_iterations</font><font color="4444FF">;</font>
-<a name="line475">475</a>       <strong>double</strong> <font color="#2040a0">final_res_norm</font><font color="4444FF">;</font>
-<a name="line476">476</a>       <strong>int</strong>    <font color="#2040a0">restart</font> <font color="4444FF">=</font> <font color="#FF0000">30</font><font color="4444FF">;</font>
-<a name="line477">477</a>       <strong>int</strong>    <font color="#2040a0">modify</font> <font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">;</font>
-<a name="line478">478</a> 
-<a name="line479">479</a> 
-<a name="line480">480</a>       <font color="#444444">/* Create solver */</font>
-<a name="line481">481</a>       <font color="#2040a0">HYPRE_ParCSRFlexGMRESCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line482">482</a> 
-<a name="line483">483</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
-<a name="line484">484</a>       <font color="#2040a0">HYPRE_FlexGMRESSetKDim</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">restart</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line485">485</a>       <font color="#2040a0">HYPRE_FlexGMRESSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* max iterations */</font>
-<a name="line486">486</a>       <font color="#2040a0">HYPRE_FlexGMRESSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance */</font>
-<a name="line487">487</a>       <font color="#2040a0">HYPRE_FlexGMRESSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print solve info */</font>
-<a name="line488">488</a>       <font color="#2040a0">HYPRE_FlexGMRESSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* needed to get run info later */</font>
-<a name="line489">489</a> 
-<a name="line490">490</a> 
-<a name="line491">491</a>       <font color="#444444">/* Now set up the AMG preconditioner and specify any parameters */</font>
-<a name="line492">492</a>       <font color="#2040a0">HYPRE_BoomerAMGCreate</font><font color="4444FF">(</font><font color="4444FF">&</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line493">493</a>       <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print amg solution info */</font>
-<a name="line494">494</a>       <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line495">495</a>       <font color="#2040a0">HYPRE_BoomerAMGSetRelaxType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* Sym G.S./Jacobi hybrid */</font>
-<a name="line496">496</a>       <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line497">497</a>       <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">0.0</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance zero */</font>
-<a name="line498">498</a>       <font color="#2040a0">HYPRE_BoomerAMGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* do only one iteration! */</font>
-<a name="line499">499</a> 
-<a name="line500">500</a>       <font color="#444444">/* Set the FlexGMRES preconditioner */</font>
-<a name="line501">501</a>       <font color="#2040a0">HYPRE_FlexGMRESSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSolve</font>,
-<a name="line502">502</a>                           <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSetup</font>, <font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line503">503</a> 
-<a name="line504">504</a> 
-<a name="line505">505</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">modify</font><font color="4444FF">)</font>
-<a name="line506">506</a>       <font color="#444444">/* this is an optional call  - if you don't call it, hypre_FlexGMRESModifyPCDefault
-<a name="line507">507</a>          is used - which does nothing.  Otherwise, you can define your own, similar to
-<a name="line508">508</a>          the one used here */</font>
-<a name="line509">509</a>          <font color="#2040a0">HYPRE_FlexGMRESSetModifyPC</font><font color="4444FF">(</font> <font color="#2040a0">solver</font>,
-<a name="line510">510</a>                                      <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToModifyPCFcn</font><font color="4444FF">)</font> <font color="#2040a0">hypre_FlexGMRESModifyPCAMGExample</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line511">511</a> 
-<a name="line512">512</a> 
-<a name="line513">513</a>       <font color="#444444">/* Now setup and solve! */</font>
-<a name="line514">514</a>       <font color="#2040a0">HYPRE_ParCSRFlexGMRESSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line515">515</a>       <font color="#2040a0">HYPRE_ParCSRFlexGMRESSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line516">516</a> 
-<a name="line517">517</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
-<a name="line518">518</a>       <font color="#2040a0">HYPRE_FlexGMRESGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line519">519</a>       <font color="#2040a0">HYPRE_FlexGMRESGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line520">520</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
-<a name="line521">521</a>       <font color="4444FF"><strong>{</strong></font>
-<a name="line522">522</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line523">523</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %lld<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line524">524</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line313">313</a>       <font color="#2040a0">HYPRE_BoomerAMGSetRelaxOrder</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>   <font color="#444444">/* Uses C/F relaxation */</font>
+<a name="line314">314</a>       <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>   <font color="#444444">/* Sweeeps on each level */</font>
+<a name="line315">315</a>       <font color="#2040a0">HYPRE_BoomerAMGSetMaxLevels</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">20</font><font color="4444FF">)</font><font color="4444FF">;</font>  <font color="#444444">/* maximum number of levels */</font>
+<a name="line316">316</a>       <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font>      <font color="#444444">/* conv. tolerance */</font>
+<a name="line317">317</a> 
+<a name="line318">318</a>       <font color="#444444">/* Now setup and solve! */</font>
+<a name="line319">319</a>       <font color="#2040a0">HYPRE_BoomerAMGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line320">320</a>       <font color="#2040a0">HYPRE_BoomerAMGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line321">321</a> 
+<a name="line322">322</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
+<a name="line323">323</a>       <font color="#2040a0">HYPRE_BoomerAMGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line324">324</a>       <font color="#2040a0">HYPRE_BoomerAMGGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line325">325</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
+<a name="line326">326</a>       <font color="4444FF"><strong>{</strong></font>
+<a name="line327">327</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line328">328</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %lld<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line329">329</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line330">330</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line331">331</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line332">332</a> 
+<a name="line333">333</a>       <font color="#444444">/* Destroy solver */</font>
+<a name="line334">334</a>       <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line335">335</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line336">336</a>    <font color="#444444">/* PCG */</font>
+<a name="line337">337</a>    <strong>else</strong> <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">50</font><font color="4444FF">)</font>
+<a name="line338">338</a>    <font color="4444FF"><strong>{</strong></font>
+<a name="line339">339</a>       <font color="#2040a0">HYPRE_Int</font> <font color="#2040a0">num_iterations</font><font color="4444FF">;</font>
+<a name="line340">340</a>       <strong>double</strong> <font color="#2040a0">final_res_norm</font><font color="4444FF">;</font>
+<a name="line341">341</a> 
+<a name="line342">342</a>       <font color="#444444">/* Create solver */</font>
+<a name="line343">343</a>       <font color="#2040a0">HYPRE_ParCSRPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line344">344</a> 
+<a name="line345">345</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
+<a name="line346">346</a>       <font color="#2040a0">HYPRE_PCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* max iterations */</font>
+<a name="line347">347</a>       <font color="#2040a0">HYPRE_PCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance */</font>
+<a name="line348">348</a>       <font color="#2040a0">HYPRE_PCGSetTwoNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* use the two norm as the stopping criteria */</font>
+<a name="line349">349</a>       <font color="#2040a0">HYPRE_PCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* prints out the iteration info */</font>
+<a name="line350">350</a>       <font color="#2040a0">HYPRE_PCGSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* needed to get run info later */</font>
+<a name="line351">351</a> 
+<a name="line352">352</a>       <font color="#444444">/* Now setup and solve! */</font>
+<a name="line353">353</a>       <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line354">354</a>       <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line355">355</a> 
+<a name="line356">356</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
+<a name="line357">357</a>       <font color="#2040a0">HYPRE_PCGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line358">358</a>       <font color="#2040a0">HYPRE_PCGGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line359">359</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
+<a name="line360">360</a>       <font color="4444FF"><strong>{</strong></font>
+<a name="line361">361</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line362">362</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %lld<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line363">363</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line364">364</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line365">365</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line366">366</a> 
+<a name="line367">367</a>       <font color="#444444">/* Destroy solver */</font>
+<a name="line368">368</a>       <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line369">369</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line370">370</a>    <font color="#444444">/* PCG with AMG preconditioner */</font>
+<a name="line371">371</a>    <strong>else</strong> <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">)</font>
+<a name="line372">372</a>    <font color="4444FF"><strong>{</strong></font>
+<a name="line373">373</a>       <font color="#2040a0">HYPRE_Int</font> <font color="#2040a0">num_iterations</font><font color="4444FF">;</font>
+<a name="line374">374</a>       <strong>double</strong> <font color="#2040a0">final_res_norm</font><font color="4444FF">;</font>
+<a name="line375">375</a> 
+<a name="line376">376</a>       <font color="#444444">/* Create solver */</font>
+<a name="line377">377</a>       <font color="#2040a0">HYPRE_ParCSRPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line378">378</a> 
+<a name="line379">379</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
+<a name="line380">380</a>       <font color="#2040a0">HYPRE_PCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* max iterations */</font>
+<a name="line381">381</a>       <font color="#2040a0">HYPRE_PCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance */</font>
+<a name="line382">382</a>       <font color="#2040a0">HYPRE_PCGSetTwoNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* use the two norm as the stopping criteria */</font>
+<a name="line383">383</a>       <font color="#2040a0">HYPRE_PCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print solve info */</font>
+<a name="line384">384</a>       <font color="#2040a0">HYPRE_PCGSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* needed to get run info later */</font>
+<a name="line385">385</a> 
+<a name="line386">386</a>       <font color="#444444">/* Now set up the AMG preconditioner and specify any parameters */</font>
+<a name="line387">387</a>       <font color="#2040a0">HYPRE_BoomerAMGCreate</font><font color="4444FF">(</font><font color="4444FF">&</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line388">388</a>       <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print amg solution info */</font>
+<a name="line389">389</a>       <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line390">390</a>       <font color="#2040a0">HYPRE_BoomerAMGSetOldDefault</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line391">391</a>       <font color="#2040a0">HYPRE_BoomerAMGSetRelaxType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* Sym G.S./Jacobi hybrid */</font>
+<a name="line392">392</a>       <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line393">393</a>       <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">0.0</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance zero */</font>
+<a name="line394">394</a>       <font color="#2040a0">HYPRE_BoomerAMGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* do only one iteration! */</font>
+<a name="line395">395</a> 
+<a name="line396">396</a>       <font color="#444444">/* Set the PCG preconditioner */</font>
+<a name="line397">397</a>       <font color="#2040a0">HYPRE_PCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSolve</font>,
+<a name="line398">398</a>                           <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSetup</font>, <font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line399">399</a> 
+<a name="line400">400</a>       <font color="#444444">/* Now setup and solve! */</font>
+<a name="line401">401</a>       <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line402">402</a>       <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line403">403</a> 
+<a name="line404">404</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
+<a name="line405">405</a>       <font color="#2040a0">HYPRE_PCGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line406">406</a>       <font color="#2040a0">HYPRE_PCGGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line407">407</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
+<a name="line408">408</a>       <font color="4444FF"><strong>{</strong></font>
+<a name="line409">409</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line410">410</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %lld<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line411">411</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line412">412</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line413">413</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line414">414</a> 
+<a name="line415">415</a>       <font color="#444444">/* Destroy solver and preconditioner */</font>
+<a name="line416">416</a>       <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line417">417</a>       <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line418">418</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line419">419</a>    <font color="#444444">/* PCG with Parasails Preconditioner */</font>
+<a name="line420">420</a>    <strong>else</strong> <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">8</font><font color="4444FF">)</font>
+<a name="line421">421</a>    <font color="4444FF"><strong>{</strong></font>
+<a name="line422">422</a>       <font color="#2040a0">HYPRE_Int</font> <font color="#2040a0">num_iterations</font><font color="4444FF">;</font>
+<a name="line423">423</a>       <strong>double</strong> <font color="#2040a0">final_res_norm</font><font color="4444FF">;</font>
+<a name="line424">424</a> 
+<a name="line425">425</a>       <strong>int</strong>      <font color="#2040a0">sai_max_levels</font> <font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">;</font>
+<a name="line426">426</a>       <strong>double</strong>   <font color="#2040a0">sai_threshold</font> <font color="4444FF">=</font> <font color="#FF0000">0.1</font><font color="4444FF">;</font>
+<a name="line427">427</a>       <strong>double</strong>   <font color="#2040a0">sai_filter</font> <font color="4444FF">=</font> <font color="#FF0000">0.05</font><font color="4444FF">;</font>
+<a name="line428">428</a>       <strong>int</strong>      <font color="#2040a0">sai_sym</font> <font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">;</font>
+<a name="line429">429</a> 
+<a name="line430">430</a>       <font color="#444444">/* Create solver */</font>
+<a name="line431">431</a>       <font color="#2040a0">HYPRE_ParCSRPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line432">432</a> 
+<a name="line433">433</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
+<a name="line434">434</a>       <font color="#2040a0">HYPRE_PCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* max iterations */</font>
+<a name="line435">435</a>       <font color="#2040a0">HYPRE_PCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance */</font>
+<a name="line436">436</a>       <font color="#2040a0">HYPRE_PCGSetTwoNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* use the two norm as the stopping criteria */</font>
+<a name="line437">437</a>       <font color="#2040a0">HYPRE_PCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print solve info */</font>
+<a name="line438">438</a>       <font color="#2040a0">HYPRE_PCGSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* needed to get run info later */</font>
+<a name="line439">439</a> 
+<a name="line440">440</a>       <font color="#444444">/* Now set up the ParaSails preconditioner and specify any parameters */</font>
+<a name="line441">441</a>       <font color="#2040a0">HYPRE_ParaSailsCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line442">442</a> 
+<a name="line443">443</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
+<a name="line444">444</a>       <font color="#2040a0">HYPRE_ParaSailsSetParams</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">sai_threshold</font>, <font color="#2040a0">sai_max_levels</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line445">445</a>       <font color="#2040a0">HYPRE_ParaSailsSetFilter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">sai_filter</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line446">446</a>       <font color="#2040a0">HYPRE_ParaSailsSetSym</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">sai_sym</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line447">447</a>       <font color="#2040a0">HYPRE_ParaSailsSetLogging</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">3</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line448">448</a> 
+<a name="line449">449</a>       <font color="#444444">/* Set the PCG preconditioner */</font>
+<a name="line450">450</a>       <font color="#2040a0">HYPRE_PCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_ParaSailsSolve</font>,
+<a name="line451">451</a>                           <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_ParaSailsSetup</font>, <font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line452">452</a> 
+<a name="line453">453</a>       <font color="#444444">/* Now setup and solve! */</font>
+<a name="line454">454</a>       <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line455">455</a>       <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line456">456</a> 
+<a name="line457">457</a> 
+<a name="line458">458</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
+<a name="line459">459</a>       <font color="#2040a0">HYPRE_PCGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line460">460</a>       <font color="#2040a0">HYPRE_PCGGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line461">461</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
+<a name="line462">462</a>       <font color="4444FF"><strong>{</strong></font>
+<a name="line463">463</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line464">464</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %lld<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line465">465</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line466">466</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line467">467</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line468">468</a> 
+<a name="line469">469</a>       <font color="#444444">/* Destory solver and preconditioner */</font>
+<a name="line470">470</a>       <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line471">471</a>       <font color="#2040a0">HYPRE_ParaSailsDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line472">472</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line473">473</a>    <font color="#444444">/* Flexible GMRES with  AMG Preconditioner */</font>
+<a name="line474">474</a>    <strong>else</strong> <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">61</font><font color="4444FF">)</font>
+<a name="line475">475</a>    <font color="4444FF"><strong>{</strong></font>
+<a name="line476">476</a>       <font color="#2040a0">HYPRE_Int</font> <font color="#2040a0">num_iterations</font><font color="4444FF">;</font>
+<a name="line477">477</a>       <strong>double</strong> <font color="#2040a0">final_res_norm</font><font color="4444FF">;</font>
+<a name="line478">478</a>       <strong>int</strong>    <font color="#2040a0">restart</font> <font color="4444FF">=</font> <font color="#FF0000">30</font><font color="4444FF">;</font>
+<a name="line479">479</a>       <strong>int</strong>    <font color="#2040a0">modify</font> <font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">;</font>
+<a name="line480">480</a> 
+<a name="line481">481</a> 
+<a name="line482">482</a>       <font color="#444444">/* Create solver */</font>
+<a name="line483">483</a>       <font color="#2040a0">HYPRE_ParCSRFlexGMRESCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="4444FF">&</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line484">484</a> 
+<a name="line485">485</a>       <font color="#444444">/* Set some parameters (See Reference Manual for more parameters) */</font>
+<a name="line486">486</a>       <font color="#2040a0">HYPRE_FlexGMRESSetKDim</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">restart</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line487">487</a>       <font color="#2040a0">HYPRE_FlexGMRESSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* max iterations */</font>
+<a name="line488">488</a>       <font color="#2040a0">HYPRE_FlexGMRESSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1e-7</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance */</font>
+<a name="line489">489</a>       <font color="#2040a0">HYPRE_FlexGMRESSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print solve info */</font>
+<a name="line490">490</a>       <font color="#2040a0">HYPRE_FlexGMRESSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* needed to get run info later */</font>
+<a name="line491">491</a> 
+<a name="line492">492</a> 
+<a name="line493">493</a>       <font color="#444444">/* Now set up the AMG preconditioner and specify any parameters */</font>
+<a name="line494">494</a>       <font color="#2040a0">HYPRE_BoomerAMGCreate</font><font color="4444FF">(</font><font color="4444FF">&</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line495">495</a>       <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* print amg solution info */</font>
+<a name="line496">496</a>       <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line497">497</a>       <font color="#2040a0">HYPRE_BoomerAMGSetOldDefault</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line498">498</a>       <font color="#2040a0">HYPRE_BoomerAMGSetRelaxType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* Sym G.S./Jacobi hybrid */</font>
+<a name="line499">499</a>       <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line500">500</a>       <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">0.0</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* conv. tolerance zero */</font>
+<a name="line501">501</a>       <font color="#2040a0">HYPRE_BoomerAMGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font> <font color="#444444">/* do only one iteration! */</font>
+<a name="line502">502</a> 
+<a name="line503">503</a>       <font color="#444444">/* Set the FlexGMRES preconditioner */</font>
+<a name="line504">504</a>       <font color="#2040a0">HYPRE_FlexGMRESSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSolve</font>,
+<a name="line505">505</a>                           <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToSolverFcn</font><font color="4444FF">)</font> <font color="#2040a0">HYPRE_BoomerAMGSetup</font>, <font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line506">506</a> 
+<a name="line507">507</a> 
+<a name="line508">508</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">modify</font><font color="4444FF">)</font>
+<a name="line509">509</a>       <font color="#444444">/* this is an optional call  - if you don't call it, hypre_FlexGMRESModifyPCDefault
+<a name="line510">510</a>          is used - which does nothing.  Otherwise, you can define your own, similar to
+<a name="line511">511</a>          the one used here */</font>
+<a name="line512">512</a>          <font color="#2040a0">HYPRE_FlexGMRESSetModifyPC</font><font color="4444FF">(</font> <font color="#2040a0">solver</font>,
+<a name="line513">513</a>                                      <font color="4444FF">(</font><font color="#2040a0">HYPRE_PtrToModifyPCFcn</font><font color="4444FF">)</font> <font color="#2040a0">hypre_FlexGMRESModifyPCAMGExample</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line514">514</a> 
+<a name="line515">515</a> 
+<a name="line516">516</a>       <font color="#444444">/* Now setup and solve! */</font>
+<a name="line517">517</a>       <font color="#2040a0">HYPRE_ParCSRFlexGMRESSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line518">518</a>       <font color="#2040a0">HYPRE_ParCSRFlexGMRESSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line519">519</a> 
+<a name="line520">520</a>       <font color="#444444">/* Run info - needed logging turned on */</font>
+<a name="line521">521</a>       <font color="#2040a0">HYPRE_FlexGMRESGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line522">522</a>       <font color="#2040a0">HYPRE_FlexGMRESGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line523">523</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
+<a name="line524">524</a>       <font color="4444FF"><strong>{</strong></font>
 <a name="line525">525</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line526">526</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line527">527</a> 
-<a name="line528">528</a>       <font color="#444444">/* Destory solver and preconditioner */</font>
-<a name="line529">529</a>       <font color="#2040a0">HYPRE_ParCSRFlexGMRESDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line530">530</a>       <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line531">531</a> 
-<a name="line532">532</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line533">533</a>    <strong>else</strong>
-<a name="line534">534</a>    <font color="4444FF"><strong>{</strong></font>
-<a name="line535">535</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font><font color="#FF0000">0</font><font color="4444FF">)</font> <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Invalid solver id specified.<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line536">536</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line537">537</a> 
-<a name="line538">538</a>    <font color="#444444">/* Clean up */</font>
-<a name="line539">539</a>    <font color="#2040a0">HYPRE_IJMatrixDestroy</font><font color="4444FF">(</font><font color="#2040a0">A</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line540">540</a>    <font color="#2040a0">HYPRE_IJVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">b</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line541">541</a>    <font color="#2040a0">HYPRE_IJVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line542">542</a> 
-<a name="line543">543</a>    <font color="#444444">/* Finalize MPI*/</font>
-<a name="line544">544</a>    <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line526">526</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %lld<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">num_iterations</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line527">527</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line528">528</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line529">529</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line530">530</a> 
+<a name="line531">531</a>       <font color="#444444">/* Destory solver and preconditioner */</font>
+<a name="line532">532</a>       <font color="#2040a0">HYPRE_ParCSRFlexGMRESDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line533">533</a>       <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line534">534</a> 
+<a name="line535">535</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line536">536</a>    <strong>else</strong>
+<a name="line537">537</a>    <font color="4444FF"><strong>{</strong></font>
+<a name="line538">538</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font><font color="#FF0000">0</font><font color="4444FF">)</font> <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Invalid solver id specified.<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line539">539</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line540">540</a> 
+<a name="line541">541</a>    <font color="#444444">/* Clean up */</font>
+<a name="line542">542</a>    <font color="#2040a0">HYPRE_IJMatrixDestroy</font><font color="4444FF">(</font><font color="#2040a0">A</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line543">543</a>    <font color="#2040a0">HYPRE_IJVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">b</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line544">544</a>    <font color="#2040a0">HYPRE_IJVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">x</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line545">545</a> 
-<a name="line546">546</a>    <strong>return</strong><font color="4444FF">(</font><font color="#FF0000">0</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line547">547</a> <font color="4444FF"><strong>}</strong></font>
+<a name="line546">546</a>    <font color="#444444">/* Finalize MPI*/</font>
+<a name="line547">547</a>    <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line548">548</a> 
-<a name="line549">549</a> <font color="#444444">/*--------------------------------------------------------------------------
-<a name="line550">550</a>    hypre_FlexGMRESModifyPCAMGExample -
+<a name="line549">549</a>    <strong>return</strong><font color="4444FF">(</font><font color="#FF0000">0</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line550">550</a> <font color="4444FF"><strong>}</strong></font>
 <a name="line551">551</a> 
-<a name="line552">552</a>     This is an example (not recommended)
-<a name="line553">553</a>    of how we can modify things about AMG that
-<a name="line554">554</a>    affect the solve phase based on how FlexGMRES is doing...For
-<a name="line555">555</a>    another preconditioner it may make sense to modify the tolerance..
-<a name="line556">556</a> 
-<a name="line557">557</a>  *--------------------------------------------------------------------------*/</font>
-<a name="line558">558</a> 
-<a name="line559">559</a> <strong>int</strong> <font color="#2040a0">hypre_FlexGMRESModifyPCAMGExample</font><font color="4444FF">(</font><strong>void</strong> <font color="4444FF">*</font><font color="#2040a0">precond_data</font>, <strong>int</strong> <font color="#2040a0">iterations</font>,
-<a name="line560">560</a>                                    <strong>double</strong> <font color="#2040a0">rel_residual_norm</font><font color="4444FF">)</font>
-<a name="line561">561</a> <font color="4444FF"><strong>{</strong></font>
-<a name="line562">562</a> 
-<a name="line563">563</a> 
-<a name="line564">564</a>    <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">rel_residual_norm</font> <font color="4444FF">></font> .<font color="#FF0000">1</font><font color="4444FF">)</font>
-<a name="line565">565</a>    <font color="4444FF"><strong>{</strong></font>
-<a name="line566">566</a> 	   <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="4444FF">(</font><font color="#2040a0">HYPRE_Solver</font><font color="4444FF">)</font><font color="#2040a0">precond_data</font>, <font color="#FF0000">10</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line567">567</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line568">568</a>    <strong>else</strong>
-<a name="line569">569</a>    <font color="4444FF"><strong>{</strong></font>
-<a name="line570">570</a> 	   <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="4444FF">(</font><font color="#2040a0">HYPRE_Solver</font><font color="4444FF">)</font><font color="#2040a0">precond_data</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line571">571</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line572">572</a> 
-<a name="line573">573</a> 
-<a name="line574">574</a>    <strong>return</strong> <font color="#FF0000">0</font><font color="4444FF">;</font>
-<a name="line575">575</a> <font color="4444FF"><strong>}</strong></font>
+<a name="line552">552</a> <font color="#444444">/*--------------------------------------------------------------------------
+<a name="line553">553</a>    hypre_FlexGMRESModifyPCAMGExample -
+<a name="line554">554</a> 
+<a name="line555">555</a>     This is an example (not recommended)
+<a name="line556">556</a>    of how we can modify things about AMG that
+<a name="line557">557</a>    affect the solve phase based on how FlexGMRES is doing...For
+<a name="line558">558</a>    another preconditioner it may make sense to modify the tolerance..
+<a name="line559">559</a> 
+<a name="line560">560</a>  *--------------------------------------------------------------------------*/</font>
+<a name="line561">561</a> 
+<a name="line562">562</a> <strong>int</strong> <font color="#2040a0">hypre_FlexGMRESModifyPCAMGExample</font><font color="4444FF">(</font><strong>void</strong> <font color="4444FF">*</font><font color="#2040a0">precond_data</font>, <strong>int</strong> <font color="#2040a0">iterations</font>,
+<a name="line563">563</a>                                    <strong>double</strong> <font color="#2040a0">rel_residual_norm</font><font color="4444FF">)</font>
+<a name="line564">564</a> <font color="4444FF"><strong>{</strong></font>
+<a name="line565">565</a> 
+<a name="line566">566</a> 
+<a name="line567">567</a>    <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">rel_residual_norm</font> <font color="4444FF">></font> .<font color="#FF0000">1</font><font color="4444FF">)</font>
+<a name="line568">568</a>    <font color="4444FF"><strong>{</strong></font>
+<a name="line569">569</a> 	   <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="4444FF">(</font><font color="#2040a0">HYPRE_Solver</font><font color="4444FF">)</font><font color="#2040a0">precond_data</font>, <font color="#FF0000">10</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line570">570</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line571">571</a>    <strong>else</strong>
+<a name="line572">572</a>    <font color="4444FF"><strong>{</strong></font>
+<a name="line573">573</a> 	   <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="4444FF">(</font><font color="#2040a0">HYPRE_Solver</font><font color="4444FF">)</font><font color="#2040a0">precond_data</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line574">574</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line575">575</a> 
+<a name="line576">576</a> 
+<a name="line577">577</a>    <strong>return</strong> <font color="#FF0000">0</font><font color="4444FF">;</font>
+<a name="line578">578</a> <font color="4444FF"><strong>}</strong></font>
 
 </pre>
 <hr>
diff --git a/src/examples/README_files/ex5f.f.html b/src/examples/README_files/ex5f.f.html
index 03034ad..1f506ee 100644
--- a/src/examples/README_files/ex5f.f.html
+++ b/src/examples/README_files/ex5f.f.html
@@ -237,231 +237,235 @@
 <a name="line230">230</a> 
 <a name="line231">231</a> <font color="#444444">c        print solve info + parameters </font>
 <a name="line232">232</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">3</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>  
-<a name="line233">233</a> <font color="#444444">c        Falgout coarsening</font>
-<a name="line234">234</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">6</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font> 
+<a name="line233">233</a> <font color="#444444">c        old defaults, Falgout coarsening, mod. class. interpolation</font>
+<a name="line234">234</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetOldDefault</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font> 
 <a name="line235">235</a> <font color="#444444">c        G-S/Jacobi hybrid relaxation </font>
 <a name="line236">236</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetRelaxType</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">3</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>     
-<a name="line237">237</a> <font color="#444444">c        Sweeeps on each level</font>
-<a name="line238">238</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>  
-<a name="line239">239</a> <font color="#444444">c         maximum number of levels </font>
-<a name="line240">240</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetMaxLevels</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">20</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font> 
-<a name="line241">241</a> <font color="#444444">c        conv. tolerance</font>
-<a name="line242">242</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1.0d-7</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>    
-<a name="line243">243</a> 
-<a name="line244">244</a> <font color="#444444">c        Now setup and solve!</font>
-<a name="line245">245</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetup</font><font color="4444FF">(</font>
-<a name="line246">246</a>      <font color="#FF0000">1</font>        <font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line247">247</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSolve</font><font color="4444FF">(</font>
+<a name="line237">237</a> <font color="#444444">c        C/F relaxation </font>
+<a name="line238">238</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetRelaxOrder</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>     
+<a name="line239">239</a> <font color="#444444">c        Sweeeps on each level</font>
+<a name="line240">240</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>  
+<a name="line241">241</a> <font color="#444444">c         maximum number of levels </font>
+<a name="line242">242</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetMaxLevels</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">20</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font> 
+<a name="line243">243</a> <font color="#444444">c        conv. tolerance</font>
+<a name="line244">244</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1.0d-7</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>    
+<a name="line245">245</a> 
+<a name="line246">246</a> <font color="#444444">c        Now setup and solve!</font>
+<a name="line247">247</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetup</font><font color="4444FF">(</font>
 <a name="line248">248</a>      <font color="#FF0000">1</font>        <font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line249">249</a> 
-<a name="line250">250</a> 
-<a name="line251">251</a> <font color="#444444">c        Run info - needed logging turned on </font>
-<a name="line252">252</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">num_iterations</font>, 
-<a name="line253">253</a>      <font color="#FF0000">1</font>        <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line254">254</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGGetFinalReltvRes</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">final_res_norm</font>,
+<a name="line249">249</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSolve</font><font color="4444FF">(</font>
+<a name="line250">250</a>      <font color="#FF0000">1</font>        <font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line251">251</a> 
+<a name="line252">252</a> 
+<a name="line253">253</a> <font color="#444444">c        Run info - needed logging turned on </font>
+<a name="line254">254</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">num_iterations</font>, 
 <a name="line255">255</a>      <font color="#FF0000">1</font>        <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line256">256</a> 
-<a name="line257">257</a> 
-<a name="line258">258</a>          <strong>if</strong> <font color="4444FF">(</font> <font color="#2040a0">myid</font> .<font color="#2040a0">eq</font>. <font color="#FF0000">0</font> <font color="4444FF">)</font> <strong>then</strong>
-<a name="line259">259</a>             <strong>print</strong> <font color="4444FF">*</font>
-<a name="line260">260</a>             <strong>print</strong> '<font color="4444FF">(</font><font color="#2040a0">A</font>,<font color="#2040a0">I2</font><font color="4444FF">)</font>', <font color="#008000">" Iterations = "</font>, <font color="#2040a0">num_iterations</font>
-<a name="line261">261</a>             <strong>print</strong> '<font color="4444FF">(</font><font color="#2040a0">A</font>,<font color="#2040a0">ES16</font><font color="#FF0000">.8</font><font color="4444FF">)</font>',
-<a name="line262">262</a>      <font color="#FF0000">1</font>            <font color="#008000">" Final Relative Residual Norm = "</font>, <font color="#2040a0">final_res_norm</font>
-<a name="line263">263</a>             <strong>print</strong> <font color="4444FF">*</font>
-<a name="line264">264</a>          <strong>endif</strong>
-<a name="line265">265</a>          
-<a name="line266">266</a> <font color="#444444">c        Destroy solver</font>
-<a name="line267">267</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line268">268</a> 
-<a name="line269">269</a> <font color="#444444">c     PCG (with DS)</font>
-<a name="line270">270</a>       <strong>elseif</strong> <font color="4444FF">(</font> <font color="#2040a0">solver_id</font> .<font color="#2040a0">eq</font>. <font color="#FF0000">50</font> <font color="4444FF">)</font> <strong>then</strong>  
-<a name="line271">271</a>          
-<a name="line272">272</a> 
-<a name="line273">273</a> <font color="#444444">c        Create solver</font>
-<a name="line274">274</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="#2040a0">solver</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line275">275</a> 
-<a name="line276">276</a> <font color="#444444">c        Set some parameters (See Reference Manual for more parameters) </font>
-<a name="line277">277</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line278">278</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1.0d-7</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line279">279</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetTwoNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line280">280</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line281">281</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line282">282</a> 
-<a name="line283">283</a> <font color="#444444">c        set ds (diagonal scaling) as the pcg preconditioner </font>
-<a name="line284">284</a>          <font color="#2040a0">precond_id</font> <font color="4444FF">=</font> <font color="#FF0000">1</font>
-<a name="line285">285</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">precond_id</font>,
-<a name="line286">286</a>      <font color="#FF0000">1</font>        <font color="#2040a0">precond</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line287">287</a> 
-<a name="line288">288</a> 
+<a name="line256">256</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGGetFinalReltvRes</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">final_res_norm</font>,
+<a name="line257">257</a>      <font color="#FF0000">1</font>        <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line258">258</a> 
+<a name="line259">259</a> 
+<a name="line260">260</a>          <strong>if</strong> <font color="4444FF">(</font> <font color="#2040a0">myid</font> .<font color="#2040a0">eq</font>. <font color="#FF0000">0</font> <font color="4444FF">)</font> <strong>then</strong>
+<a name="line261">261</a>             <strong>print</strong> <font color="4444FF">*</font>
+<a name="line262">262</a>             <strong>print</strong> '<font color="4444FF">(</font><font color="#2040a0">A</font>,<font color="#2040a0">I2</font><font color="4444FF">)</font>', <font color="#008000">" Iterations = "</font>, <font color="#2040a0">num_iterations</font>
+<a name="line263">263</a>             <strong>print</strong> '<font color="4444FF">(</font><font color="#2040a0">A</font>,<font color="#2040a0">ES16</font><font color="#FF0000">.8</font><font color="4444FF">)</font>',
+<a name="line264">264</a>      <font color="#FF0000">1</font>            <font color="#008000">" Final Relative Residual Norm = "</font>, <font color="#2040a0">final_res_norm</font>
+<a name="line265">265</a>             <strong>print</strong> <font color="4444FF">*</font>
+<a name="line266">266</a>          <strong>endif</strong>
+<a name="line267">267</a>          
+<a name="line268">268</a> <font color="#444444">c        Destroy solver</font>
+<a name="line269">269</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line270">270</a> 
+<a name="line271">271</a> <font color="#444444">c     PCG (with DS)</font>
+<a name="line272">272</a>       <strong>elseif</strong> <font color="4444FF">(</font> <font color="#2040a0">solver_id</font> .<font color="#2040a0">eq</font>. <font color="#FF0000">50</font> <font color="4444FF">)</font> <strong>then</strong>  
+<a name="line273">273</a>          
+<a name="line274">274</a> 
+<a name="line275">275</a> <font color="#444444">c        Create solver</font>
+<a name="line276">276</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="#2040a0">solver</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line277">277</a> 
+<a name="line278">278</a> <font color="#444444">c        Set some parameters (See Reference Manual for more parameters) </font>
+<a name="line279">279</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line280">280</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1.0d-7</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line281">281</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetTwoNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line282">282</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line283">283</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line284">284</a> 
+<a name="line285">285</a> <font color="#444444">c        set ds (diagonal scaling) as the pcg preconditioner </font>
+<a name="line286">286</a>          <font color="#2040a0">precond_id</font> <font color="4444FF">=</font> <font color="#FF0000">1</font>
+<a name="line287">287</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">precond_id</font>,
+<a name="line288">288</a>      <font color="#FF0000">1</font>        <font color="#2040a0">precond</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
 <a name="line289">289</a> 
-<a name="line290">290</a> <font color="#444444">c        Now setup and solve!</font>
-<a name="line291">291</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>,
-<a name="line292">292</a>      <font color="4444FF">&</font>                            <font color="#2040a0">par_x</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line293">293</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>,
+<a name="line290">290</a> 
+<a name="line291">291</a> 
+<a name="line292">292</a> <font color="#444444">c        Now setup and solve!</font>
+<a name="line293">293</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>,
 <a name="line294">294</a>      <font color="4444FF">&</font>                            <font color="#2040a0">par_x</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line295">295</a> 
-<a name="line296">296</a> 
-<a name="line297">297</a> <font color="#444444">c        Run info - needed logging turned on </font>
+<a name="line295">295</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>,
+<a name="line296">296</a>      <font color="4444FF">&</font>                            <font color="#2040a0">par_x</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line297">297</a> 
 <a name="line298">298</a> 
-<a name="line299">299</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">num_iterations</font>,
-<a name="line300">300</a>      <font color="4444FF">&</font>                                       <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line301">301</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGGetFinalRelative</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">final_res_norm</font>,
+<a name="line299">299</a> <font color="#444444">c        Run info - needed logging turned on </font>
+<a name="line300">300</a> 
+<a name="line301">301</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">num_iterations</font>,
 <a name="line302">302</a>      <font color="4444FF">&</font>                                       <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line303">303</a> 
-<a name="line304">304</a>        <strong>if</strong> <font color="4444FF">(</font> <font color="#2040a0">myid</font> .<font color="#2040a0">eq</font>. <font color="#FF0000">0</font> <font color="4444FF">)</font> <strong>then</strong>
-<a name="line305">305</a>             <strong>print</strong> <font color="4444FF">*</font>
-<a name="line306">306</a>             <strong>print</strong> <font color="4444FF">*</font>, <font color="#008000">"Iterations = "</font>, <font color="#2040a0">num_iterations</font>
-<a name="line307">307</a>             <strong>print</strong> <font color="4444FF">*</font>, <font color="#008000">"Final Relative Residual Norm = "</font>, <font color="#2040a0">final_res_norm</font>
-<a name="line308">308</a>             <strong>print</strong> <font color="4444FF">*</font>
-<a name="line309">309</a>          <strong>endif</strong>
-<a name="line310">310</a> 
-<a name="line311">311</a> <font color="#444444">c       Destroy solver </font>
-<a name="line312">312</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line313">313</a> 
-<a name="line314">314</a> 
-<a name="line315">315</a> <font color="#444444">c     PCG with AMG preconditioner</font>
-<a name="line316">316</a>       <strong>elseif</strong> <font color="4444FF">(</font> <font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">1</font> <font color="4444FF">)</font> <strong>then</strong>
-<a name="line317">317</a>      
-<a name="line318">318</a> <font color="#444444">c        Create solver</font>
-<a name="line319">319</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="#2040a0">solver</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line320">320</a> 
-<a name="line321">321</a> <font color="#444444">c        Set some parameters (See Reference Manual for more parameters) </font>
-<a name="line322">322</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line323">323</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1.0d-7</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line324">324</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetTwoNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line325">325</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line326">326</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line327">327</a> 
-<a name="line328">328</a> <font color="#444444">c        Now set up the AMG preconditioner and specify any parameters</font>
+<a name="line303">303</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGGetFinalRelative</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">final_res_norm</font>,
+<a name="line304">304</a>      <font color="4444FF">&</font>                                       <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line305">305</a> 
+<a name="line306">306</a>        <strong>if</strong> <font color="4444FF">(</font> <font color="#2040a0">myid</font> .<font color="#2040a0">eq</font>. <font color="#FF0000">0</font> <font color="4444FF">)</font> <strong>then</strong>
+<a name="line307">307</a>             <strong>print</strong> <font color="4444FF">*</font>
+<a name="line308">308</a>             <strong>print</strong> <font color="4444FF">*</font>, <font color="#008000">"Iterations = "</font>, <font color="#2040a0">num_iterations</font>
+<a name="line309">309</a>             <strong>print</strong> <font color="4444FF">*</font>, <font color="#008000">"Final Relative Residual Norm = "</font>, <font color="#2040a0">final_res_norm</font>
+<a name="line310">310</a>             <strong>print</strong> <font color="4444FF">*</font>
+<a name="line311">311</a>          <strong>endif</strong>
+<a name="line312">312</a> 
+<a name="line313">313</a> <font color="#444444">c       Destroy solver </font>
+<a name="line314">314</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line315">315</a> 
+<a name="line316">316</a> 
+<a name="line317">317</a> <font color="#444444">c     PCG with AMG preconditioner</font>
+<a name="line318">318</a>       <strong>elseif</strong> <font color="4444FF">(</font> <font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">1</font> <font color="4444FF">)</font> <strong>then</strong>
+<a name="line319">319</a>      
+<a name="line320">320</a> <font color="#444444">c        Create solver</font>
+<a name="line321">321</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="#2040a0">solver</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line322">322</a> 
+<a name="line323">323</a> <font color="#444444">c        Set some parameters (See Reference Manual for more parameters) </font>
+<a name="line324">324</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line325">325</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1.0d-7</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line326">326</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetTwoNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line327">327</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line328">328</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
 <a name="line329">329</a> 
-<a name="line330">330</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGCreate</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line330">330</a> <font color="#444444">c        Now set up the AMG preconditioner and specify any parameters</font>
 <a name="line331">331</a> 
-<a name="line332">332</a> 
-<a name="line333">333</a> <font color="#444444">c        Set some parameters (See Reference Manual for more parameters)</font>
+<a name="line332">332</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGCreate</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line333">333</a> 
 <a name="line334">334</a> 
-<a name="line335">335</a> <font color="#444444">c        print less solver info since a preconditioner</font>
-<a name="line336">336</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font><font color="4444FF">;</font> 
-<a name="line337">337</a> <font color="#444444">c        Falgout coarsening</font>
-<a name="line338">338</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font> 
-<a name="line339">339</a> <font color="#444444">c        SYMMETRIC G-S/Jacobi hybrid relaxation </font>
-<a name="line340">340</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetRelaxType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>     
-<a name="line341">341</a> <font color="#444444">c        Sweeeps on each level</font>
-<a name="line342">342</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>  
-<a name="line343">343</a> <font color="#444444">c        conv. tolerance</font>
-<a name="line344">344</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">0.0d0</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>     
-<a name="line345">345</a> <font color="#444444">c        do only one iteration! </font>
-<a name="line346">346</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line347">347</a> 
-<a name="line348">348</a> <font color="#444444">c        set amg as the pcg preconditioner</font>
-<a name="line349">349</a>          <font color="#2040a0">precond_id</font> <font color="4444FF">=</font> <font color="#FF0000">2</font>
-<a name="line350">350</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">precond_id</font>,
-<a name="line351">351</a>      <font color="#FF0000">1</font>        <font color="#2040a0">precond</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line352">352</a> 
-<a name="line353">353</a> 
-<a name="line354">354</a> <font color="#444444">c        Now setup and solve!</font>
-<a name="line355">355</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>,
-<a name="line356">356</a>      <font color="#FF0000">1</font>                            <font color="#2040a0">par_x</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line357">357</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>,
-<a name="line358">358</a>      <font color="#FF0000">1</font>                            <font color="#2040a0">par_x</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line359">359</a> 
-<a name="line360">360</a> 
-<a name="line361">361</a> <font color="#444444">c        Run info - needed logging turned on </font>
-<a name="line362">362</a> 
-<a name="line363">363</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">num_iterations</font>,
-<a name="line364">364</a>      <font color="#FF0000">1</font>                                       <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line365">365</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGGetFinalRelative</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">final_res_norm</font>,
-<a name="line366">366</a>      <font color="#FF0000">1</font>                                       <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line367">367</a> 
-<a name="line368">368</a>        <strong>if</strong> <font color="4444FF">(</font> <font color="#2040a0">myid</font> .<font color="#2040a0">eq</font>. <font color="#FF0000">0</font> <font color="4444FF">)</font> <strong>then</strong>
-<a name="line369">369</a>             <strong>print</strong> <font color="4444FF">*</font>
-<a name="line370">370</a>             <strong>print</strong> <font color="4444FF">*</font>, <font color="#008000">"Iterations = "</font>, <font color="#2040a0">num_iterations</font>
-<a name="line371">371</a>             <strong>print</strong> <font color="4444FF">*</font>, <font color="#008000">"Final Relative Residual Norm = "</font>, <font color="#2040a0">final_res_norm</font>
-<a name="line372">372</a>             <strong>print</strong> <font color="4444FF">*</font>
-<a name="line373">373</a>          <strong>endif</strong>
-<a name="line374">374</a> 
-<a name="line375">375</a> <font color="#444444">c       Destroy precond and solver</font>
-<a name="line376">376</a> 
-<a name="line377">377</a>         <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line378">378</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line379">379</a> 
-<a name="line380">380</a> <font color="#444444">c     PCG with ParaSails</font>
-<a name="line381">381</a>       <strong>elseif</strong> <font color="4444FF">(</font> <font color="#2040a0">solver_id</font> .<font color="#2040a0">eq</font>. <font color="#FF0000">8</font> <font color="4444FF">)</font> <strong>then</strong>
-<a name="line382">382</a> 
-<a name="line383">383</a> <font color="#444444">c        Create solver</font>
-<a name="line384">384</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="#2040a0">solver</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line385">385</a> 
-<a name="line386">386</a> <font color="#444444">c        Set some parameters (See Reference Manual for more parameters) </font>
-<a name="line387">387</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line388">388</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1.0d-7</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line389">389</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetTwoNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line390">390</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line391">391</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line392">392</a> 
-<a name="line393">393</a> <font color="#444444">c        Now set up the Parasails preconditioner and specify any parameters</font>
-<a name="line394">394</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParaSailsCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="#2040a0">precond</font>,<font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line395">395</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParaSailsSetParams</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">0.1d0</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line396">396</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParaSailsSetFilter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">0.05d0</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line397">397</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParaSailsSetSym</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font>
-<a name="line398">398</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParaSailsSetLogging</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">3</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line399">399</a> 
-<a name="line400">400</a> <font color="#444444">c        set parsails as the pcg preconditioner</font>
-<a name="line401">401</a>          <font color="#2040a0">precond_id</font> <font color="4444FF">=</font> <font color="#FF0000">4</font>
-<a name="line402">402</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">precond_id</font>,
-<a name="line403">403</a>      <font color="#FF0000">1</font>        <font color="#2040a0">precond</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line404">404</a> 
-<a name="line405">405</a> 
-<a name="line406">406</a> <font color="#444444">c        Now setup and solve!</font>
-<a name="line407">407</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>,
-<a name="line408">408</a>      <font color="#FF0000">1</font>                            <font color="#2040a0">par_x</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line409">409</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>,
-<a name="line410">410</a>      <font color="#FF0000">1</font>                            <font color="#2040a0">par_x</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line411">411</a> 
-<a name="line412">412</a> 
-<a name="line413">413</a> <font color="#444444">c        Run info - needed logging turned on </font>
-<a name="line414">414</a> 
-<a name="line415">415</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">num_iterations</font>,
-<a name="line416">416</a>      <font color="#FF0000">1</font>                                       <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line417">417</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGGetFinalRelative</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">final_res_norm</font>,
-<a name="line418">418</a>      <font color="#FF0000">1</font>                                       <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line419">419</a> 
-<a name="line420">420</a>        <strong>if</strong> <font color="4444FF">(</font> <font color="#2040a0">myid</font> .<font color="#2040a0">eq</font>. <font color="#FF0000">0</font> <font color="4444FF">)</font> <strong>then</strong>
-<a name="line421">421</a>             <strong>print</strong> <font color="4444FF">*</font>
-<a name="line422">422</a>             <strong>print</strong> <font color="4444FF">*</font>, <font color="#008000">"Iterations = "</font>, <font color="#2040a0">num_iterations</font>
-<a name="line423">423</a>             <strong>print</strong> <font color="4444FF">*</font>, <font color="#008000">"Final Relative Residual Norm = "</font>, <font color="#2040a0">final_res_norm</font>
-<a name="line424">424</a>             <strong>print</strong> <font color="4444FF">*</font>
-<a name="line425">425</a>          <strong>endif</strong>
-<a name="line426">426</a> 
-<a name="line427">427</a> <font color="#444444">c       Destroy precond and solver</font>
-<a name="line428">428</a> 
-<a name="line429">429</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParaSailsDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line430">430</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line431">431</a> 
-<a name="line432">432</a>       <strong>else</strong>
-<a name="line433">433</a>          <strong>if</strong> <font color="4444FF">(</font> <font color="#2040a0">myid</font> .<font color="#2040a0">eq</font>. <font color="#FF0000">0</font> <font color="4444FF">)</font> <strong>then</strong> 
-<a name="line434">434</a>            <strong>print</strong> <font color="4444FF">*</font>,'<font color="#2040a0">Invalid</font> <font color="#2040a0">solver</font> <font color="#2040a0">id</font> <font color="#2040a0">specified</font>'
-<a name="line435">435</a>            <strong>stop</strong>
-<a name="line436">436</a>          <strong>endif</strong>  
-<a name="line437">437</a>       <strong>endif</strong>
-<a name="line438">438</a> 
-<a name="line439">439</a> 
-<a name="line440">440</a> 
-<a name="line441">441</a> <font color="#444444">c     Print the solution</font>
-<a name="line442">442</a>       <strong>if</strong> <font color="4444FF">(</font> <font color="#2040a0">print_solution</font> .<font color="#2040a0">ne</font>. <font color="#FF0000">0</font> <font color="4444FF">)</font> <strong>then</strong>
-<a name="line443">443</a>          <strong>call</strong> <font color="#2040a0">HYPRE_IJVectorPrint</font><font color="4444FF">(</font><font color="#2040a0">x</font>, <font color="#008000">"ij.out.x"</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line444">444</a>       <strong>endif</strong>
-<a name="line445">445</a> 
-<a name="line446">446</a> <font color="#444444">c     Clean up</font>
-<a name="line447">447</a> 
-<a name="line448">448</a>       <strong>call</strong> <font color="#2040a0">HYPRE_IJMatrixDestroy</font><font color="4444FF">(</font><font color="#2040a0">A</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line449">449</a>       <strong>call</strong> <font color="#2040a0">HYPRE_IJVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">b</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
-<a name="line450">450</a>       <strong>call</strong> <font color="#2040a0">HYPRE_IJVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">x</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line335">335</a> <font color="#444444">c        Set some parameters (See Reference Manual for more parameters)</font>
+<a name="line336">336</a> 
+<a name="line337">337</a> <font color="#444444">c        print less solver info since a preconditioner</font>
+<a name="line338">338</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font><font color="4444FF">;</font> 
+<a name="line339">339</a> <font color="#444444">c        Falgout coarsening</font>
+<a name="line340">340</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font> 
+<a name="line341">341</a> <font color="#444444">c        old defaults</font>
+<a name="line342">342</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetOldDefault</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font> 
+<a name="line343">343</a> <font color="#444444">c        SYMMETRIC G-S/Jacobi hybrid relaxation </font>
+<a name="line344">344</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetRelaxType</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">6</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>     
+<a name="line345">345</a> <font color="#444444">c        Sweeeps on each level</font>
+<a name="line346">346</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetNumSweeps</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>  
+<a name="line347">347</a> <font color="#444444">c        conv. tolerance</font>
+<a name="line348">348</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">0.0d0</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>     
+<a name="line349">349</a> <font color="#444444">c        do only one iteration! </font>
+<a name="line350">350</a>          <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line351">351</a> 
+<a name="line352">352</a> <font color="#444444">c        set amg as the pcg preconditioner</font>
+<a name="line353">353</a>          <font color="#2040a0">precond_id</font> <font color="4444FF">=</font> <font color="#FF0000">2</font>
+<a name="line354">354</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">precond_id</font>,
+<a name="line355">355</a>      <font color="#FF0000">1</font>        <font color="#2040a0">precond</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line356">356</a> 
+<a name="line357">357</a> 
+<a name="line358">358</a> <font color="#444444">c        Now setup and solve!</font>
+<a name="line359">359</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>,
+<a name="line360">360</a>      <font color="#FF0000">1</font>                            <font color="#2040a0">par_x</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line361">361</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>,
+<a name="line362">362</a>      <font color="#FF0000">1</font>                            <font color="#2040a0">par_x</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line363">363</a> 
+<a name="line364">364</a> 
+<a name="line365">365</a> <font color="#444444">c        Run info - needed logging turned on </font>
+<a name="line366">366</a> 
+<a name="line367">367</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">num_iterations</font>,
+<a name="line368">368</a>      <font color="#FF0000">1</font>                                       <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line369">369</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGGetFinalRelative</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">final_res_norm</font>,
+<a name="line370">370</a>      <font color="#FF0000">1</font>                                       <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line371">371</a> 
+<a name="line372">372</a>        <strong>if</strong> <font color="4444FF">(</font> <font color="#2040a0">myid</font> .<font color="#2040a0">eq</font>. <font color="#FF0000">0</font> <font color="4444FF">)</font> <strong>then</strong>
+<a name="line373">373</a>             <strong>print</strong> <font color="4444FF">*</font>
+<a name="line374">374</a>             <strong>print</strong> <font color="4444FF">*</font>, <font color="#008000">"Iterations = "</font>, <font color="#2040a0">num_iterations</font>
+<a name="line375">375</a>             <strong>print</strong> <font color="4444FF">*</font>, <font color="#008000">"Final Relative Residual Norm = "</font>, <font color="#2040a0">final_res_norm</font>
+<a name="line376">376</a>             <strong>print</strong> <font color="4444FF">*</font>
+<a name="line377">377</a>          <strong>endif</strong>
+<a name="line378">378</a> 
+<a name="line379">379</a> <font color="#444444">c       Destroy precond and solver</font>
+<a name="line380">380</a> 
+<a name="line381">381</a>         <strong>call</strong> <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line382">382</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line383">383</a> 
+<a name="line384">384</a> <font color="#444444">c     PCG with ParaSails</font>
+<a name="line385">385</a>       <strong>elseif</strong> <font color="4444FF">(</font> <font color="#2040a0">solver_id</font> .<font color="#2040a0">eq</font>. <font color="#FF0000">8</font> <font color="4444FF">)</font> <strong>then</strong>
+<a name="line386">386</a> 
+<a name="line387">387</a> <font color="#444444">c        Create solver</font>
+<a name="line388">388</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="#2040a0">solver</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line389">389</a> 
+<a name="line390">390</a> <font color="#444444">c        Set some parameters (See Reference Manual for more parameters) </font>
+<a name="line391">391</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1000</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line392">392</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetTol</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1.0d-7</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line393">393</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetTwoNorm</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line394">394</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">2</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line395">395</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetLogging</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line396">396</a> 
+<a name="line397">397</a> <font color="#444444">c        Now set up the Parasails preconditioner and specify any parameters</font>
+<a name="line398">398</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParaSailsCreate</font><font color="4444FF">(</font><font color="#2040a0">MPI_COMM_WORLD</font>, <font color="#2040a0">precond</font>,<font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line399">399</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParaSailsSetParams</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">0.1d0</font>, <font color="#FF0000">1</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line400">400</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParaSailsSetFilter</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">0.05d0</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line401">401</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParaSailsSetSym</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font>
+<a name="line402">402</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParaSailsSetLogging</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#FF0000">3</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line403">403</a> 
+<a name="line404">404</a> <font color="#444444">c        set parsails as the pcg preconditioner</font>
+<a name="line405">405</a>          <font color="#2040a0">precond_id</font> <font color="4444FF">=</font> <font color="#FF0000">4</font>
+<a name="line406">406</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">precond_id</font>,
+<a name="line407">407</a>      <font color="#FF0000">1</font>        <font color="#2040a0">precond</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line408">408</a> 
+<a name="line409">409</a> 
+<a name="line410">410</a> <font color="#444444">c        Now setup and solve!</font>
+<a name="line411">411</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSetup</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>,
+<a name="line412">412</a>      <font color="#FF0000">1</font>                            <font color="#2040a0">par_x</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line413">413</a>          <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGSolve</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">parcsr_A</font>, <font color="#2040a0">par_b</font>,
+<a name="line414">414</a>      <font color="#FF0000">1</font>                            <font color="#2040a0">par_x</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line415">415</a> 
+<a name="line416">416</a> 
+<a name="line417">417</a> <font color="#444444">c        Run info - needed logging turned on </font>
+<a name="line418">418</a> 
+<a name="line419">419</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">num_iterations</font>,
+<a name="line420">420</a>      <font color="#FF0000">1</font>                                       <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line421">421</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGGetFinalRelative</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">final_res_norm</font>,
+<a name="line422">422</a>      <font color="#FF0000">1</font>                                       <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line423">423</a> 
+<a name="line424">424</a>        <strong>if</strong> <font color="4444FF">(</font> <font color="#2040a0">myid</font> .<font color="#2040a0">eq</font>. <font color="#FF0000">0</font> <font color="4444FF">)</font> <strong>then</strong>
+<a name="line425">425</a>             <strong>print</strong> <font color="4444FF">*</font>
+<a name="line426">426</a>             <strong>print</strong> <font color="4444FF">*</font>, <font color="#008000">"Iterations = "</font>, <font color="#2040a0">num_iterations</font>
+<a name="line427">427</a>             <strong>print</strong> <font color="4444FF">*</font>, <font color="#008000">"Final Relative Residual Norm = "</font>, <font color="#2040a0">final_res_norm</font>
+<a name="line428">428</a>             <strong>print</strong> <font color="4444FF">*</font>
+<a name="line429">429</a>          <strong>endif</strong>
+<a name="line430">430</a> 
+<a name="line431">431</a> <font color="#444444">c       Destroy precond and solver</font>
+<a name="line432">432</a> 
+<a name="line433">433</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParaSailsDestroy</font><font color="4444FF">(</font><font color="#2040a0">precond</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line434">434</a>         <strong>call</strong> <font color="#2040a0">HYPRE_ParCSRPCGDestroy</font><font color="4444FF">(</font><font color="#2040a0">solver</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line435">435</a> 
+<a name="line436">436</a>       <strong>else</strong>
+<a name="line437">437</a>          <strong>if</strong> <font color="4444FF">(</font> <font color="#2040a0">myid</font> .<font color="#2040a0">eq</font>. <font color="#FF0000">0</font> <font color="4444FF">)</font> <strong>then</strong> 
+<a name="line438">438</a>            <strong>print</strong> <font color="4444FF">*</font>,'<font color="#2040a0">Invalid</font> <font color="#2040a0">solver</font> <font color="#2040a0">id</font> <font color="#2040a0">specified</font>'
+<a name="line439">439</a>            <strong>stop</strong>
+<a name="line440">440</a>          <strong>endif</strong>  
+<a name="line441">441</a>       <strong>endif</strong>
+<a name="line442">442</a> 
+<a name="line443">443</a> 
+<a name="line444">444</a> 
+<a name="line445">445</a> <font color="#444444">c     Print the solution</font>
+<a name="line446">446</a>       <strong>if</strong> <font color="4444FF">(</font> <font color="#2040a0">print_solution</font> .<font color="#2040a0">ne</font>. <font color="#FF0000">0</font> <font color="4444FF">)</font> <strong>then</strong>
+<a name="line447">447</a>          <strong>call</strong> <font color="#2040a0">HYPRE_IJVectorPrint</font><font color="4444FF">(</font><font color="#2040a0">x</font>, <font color="#008000">"ij.out.x"</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line448">448</a>       <strong>endif</strong>
+<a name="line449">449</a> 
+<a name="line450">450</a> <font color="#444444">c     Clean up</font>
 <a name="line451">451</a> 
-<a name="line452">452</a> 
-<a name="line453">453</a> <font color="#444444">c     Finalize MPI</font>
-<a name="line454">454</a>       <strong>call</strong> <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line452">452</a>       <strong>call</strong> <font color="#2040a0">HYPRE_IJMatrixDestroy</font><font color="4444FF">(</font><font color="#2040a0">A</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line453">453</a>       <strong>call</strong> <font color="#2040a0">HYPRE_IJVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">b</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line454">454</a>       <strong>call</strong> <font color="#2040a0">HYPRE_IJVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">x</font>, <font color="#2040a0">ierr</font><font color="4444FF">)</font>
 <a name="line455">455</a> 
-<a name="line456">456</a>       <strong>stop</strong>
-<a name="line457">457</a>       <strong>end</strong>
+<a name="line456">456</a> 
+<a name="line457">457</a> <font color="#444444">c     Finalize MPI</font>
+<a name="line458">458</a>       <strong>call</strong> <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="#2040a0">ierr</font><font color="4444FF">)</font>
+<a name="line459">459</a> 
+<a name="line460">460</a>       <strong>stop</strong>
+<a name="line461">461</a>       <strong>end</strong>
 
 </pre>
 <hr>
diff --git a/src/examples/README_files/ex9.c.html b/src/examples/README_files/ex9.c.html
index e58f12a..fbe114d 100644
--- a/src/examples/README_files/ex9.c.html
+++ b/src/examples/README_files/ex9.c.html
@@ -636,152 +636,154 @@
 <a name="line629">629</a>          <font color="#444444">/* use BoomerAMG as preconditioner */</font>
 <a name="line630">630</a>          <font color="#2040a0">HYPRE_BoomerAMGCreate</font><font color="4444FF">(</font><font color="4444FF">&</font><font color="#2040a0">par_precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line631">631</a>          <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">par_precond</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line632">632</a>          <font color="#2040a0">HYPRE_BoomerAMGSetStrongThreshold</font><font color="4444FF">(</font><font color="#2040a0">par_precond</font>, <font color="#FF0000">0.25</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line633">633</a>          <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">par_precond</font>, <font color="#FF0000">0.0</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line634">634</a>          <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">par_precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line635">635</a>          <font color="#2040a0">HYPRE_BoomerAMGSetPrintFileName</font><font color="4444FF">(</font><font color="#2040a0">par_precond</font>, <font color="#008000">"ex9.out.log"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line636">636</a>          <font color="#2040a0">HYPRE_BoomerAMGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">par_precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line637">637</a> 
-<a name="line638">638</a>          <font color="#444444">/* set the preconditioner */</font>
-<a name="line639">639</a>          <font color="#2040a0">HYPRE_ParCSRGMRESSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>,
-<a name="line640">640</a>                                      <font color="#2040a0">HYPRE_BoomerAMGSolve</font>,
-<a name="line641">641</a>                                      <font color="#2040a0">HYPRE_BoomerAMGSetup</font>,
-<a name="line642">642</a>                                      <font color="#2040a0">par_precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line643">643</a> 
-<a name="line644">644</a>          <font color="#444444">/* do the setup */</font>
-<a name="line645">645</a>          <font color="#2040a0">HYPRE_ParCSRGMRESSetup</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#2040a0">par_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line646">646</a> 
-<a name="line647">647</a>          <font color="#444444">/* do the solve */</font>
-<a name="line648">648</a>          <font color="#2040a0">HYPRE_ParCSRGMRESSolve</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#2040a0">par_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line649">649</a> 
-<a name="line650">650</a>          <font color="#444444">/* get some info */</font>
-<a name="line651">651</a>          <font color="#2040a0">HYPRE_GMRESGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="4444FF">&</font><font color="#2040a0">its</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line652">652</a>          <font color="#2040a0">HYPRE_GMRESGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>,
-<a name="line653">653</a>                                                  <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line654">654</a>          <font color="#444444">/* clean up */</font>
-<a name="line655">655</a>          <font color="#2040a0">HYPRE_ParCSRGMRESDestroy</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line656">656</a>          <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">par_precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line657">657</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line658">658</a>       <strong>else</strong> <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">3</font><font color="4444FF">)</font> <font color="#444444">/* AMG */</font>
-<a name="line659">659</a>       <font color="4444FF"><strong>{</strong></font>
-<a name="line660">660</a>          <font color="#2040a0">HYPRE_BoomerAMGCreate</font><font color="4444FF">(</font><font color="4444FF">&</font><font color="#2040a0">par_solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line661">661</a>          <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line662">662</a>          <font color="#2040a0">HYPRE_BoomerAMGSetStrongThreshold</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#FF0000">0.25</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line663">663</a>          <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#FF0000">1.9e-6</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line664">664</a>          <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line665">665</a>          <font color="#2040a0">HYPRE_BoomerAMGSetPrintFileName</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#008000">"ex9.out.log"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line666">666</a>          <font color="#2040a0">HYPRE_BoomerAMGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#FF0000">50</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line667">667</a> 
-<a name="line668">668</a>          <font color="#444444">/* do the setup */</font>
-<a name="line669">669</a>          <font color="#2040a0">HYPRE_BoomerAMGSetup</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#2040a0">par_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line670">670</a> 
-<a name="line671">671</a>          <font color="#444444">/* do the solve */</font>
-<a name="line672">672</a>          <font color="#2040a0">HYPRE_BoomerAMGSolve</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#2040a0">par_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line673">673</a> 
-<a name="line674">674</a>          <font color="#444444">/* get some info */</font>
-<a name="line675">675</a>          <font color="#2040a0">HYPRE_BoomerAMGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="4444FF">&</font><font color="#2040a0">its</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line676">676</a>          <font color="#2040a0">HYPRE_BoomerAMGGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>,
-<a name="line677">677</a>                                                      <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line678">678</a>          <font color="#444444">/* clean up */</font>
-<a name="line679">679</a>          <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line680">680</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line681">681</a>       <strong>else</strong>
-<a name="line682">682</a>       <font color="4444FF"><strong>{</strong></font>
-<a name="line683">683</a>          <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font><font color="#FF0000">0</font><font color="4444FF">)</font> <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font> ERROR: Invalid solver id specified.<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line684">684</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line685">685</a> 
-<a name="line686">686</a>       <font color="#444444">/* Gather the solution vector.  This needs to be done if:
-<a name="line687">687</a>          (1) the  object  type is parcsr OR
-<a name="line688">688</a>          (2) any one of the variables is NOT cell-centered */</font>
-<a name="line689">689</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">object_type</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#2040a0">HYPRE_PARCSR</font><font color="4444FF">)</font>
-<a name="line690">690</a>       <font color="4444FF"><strong>{</strong></font>
-<a name="line691">691</a>          <font color="#2040a0">HYPRE_SStructVectorGather</font><font color="4444FF">(</font><font color="#2040a0">x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line692">692</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line693">693</a> 
-<a name="line694">694</a>       <font color="#444444">/* Save the solution for GLVis visualization, see vis/glvis-ex7.sh */</font>
-<a name="line695">695</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">vis</font><font color="4444FF">)</font>
-<a name="line696">696</a>       <font color="4444FF"><strong>{</strong></font>
-<a name="line697">697</a>          <font color="#2040a0">FILE</font> <font color="4444FF">*</font><font color="#2040a0">file</font><font color="4444FF">;</font>
-<a name="line698">698</a>          <strong>char</strong> <font color="#2040a0">filename</font><font color="4444FF">[</font><font color="#FF0000">255</font><font color="4444FF">]</font><font color="4444FF">;</font>
-<a name="line699">699</a> 
-<a name="line700">700</a>          <strong>int</strong> <font color="#2040a0">k</font>, <font color="#2040a0">part</font> <font color="4444FF">=</font> <font color="#FF0000">0</font>, <font color="#2040a0">var</font><font color="4444FF">;</font>
-<a name="line701">701</a>          <strong>int</strong> <font color="#2040a0">nvalues</font> <font color="4444FF">=</font> <font color="#2040a0">n</font><font color="4444FF">*</font><font color="#2040a0">n</font><font color="4444FF">;</font>
-<a name="line702">702</a>          <strong>double</strong> <font color="4444FF">*</font><font color="#2040a0">values</font> <font color="4444FF">=</font> <font color="4444FF">(</font><strong>double</strong><font color="4444FF">*</font><font color="4444FF">)</font> <font color="#2040a0">calloc</font><font color="4444FF">(</font><font color="#2040a0">nvalues</font>, <strong>sizeof</strong><font color="4444FF">(</font><strong>double</strong><font color="4444FF">)</font><font color="4444FF"> [...]
-<a name="line703">703</a> 
-<a name="line704">704</a>          <font color="#444444">/* save local solution for variable u */</font>
-<a name="line705">705</a>          <font color="#2040a0">var</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font>
-<a name="line706">706</a>          <font color="#2040a0">HYPRE_SStructVectorGetBoxValues</font><font color="4444FF">(</font><font color="#2040a0">x</font>, <font color="#2040a0">part</font>, <font color="#2040a0">ilower</font>, <font color="#2040a0">iupper</font>,
-<a name="line707">707</a>                                          <font color="#2040a0">var</font>, <font color="#2040a0">values</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line708">708</a> 
-<a name="line709">709</a>          <font color="#2040a0">sprintf</font><font color="4444FF">(</font><font color="#2040a0">filename</font>, <font color="#008000">"%s.%06d"</font>, <font color="#008000">"vis/ex9-u.sol"</font>, <font color="#2040a0">myid</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line710">710</a>          <strong>if</strong> <font color="4444FF">(</font><font color="4444FF">(</font><font color="#2040a0">file</font> <font color="4444FF">=</font> <font color="#2040a0">fopen</font><font color="4444FF">(</font><font color="#2040a0">filename</font>, <font color="#008000">"w"</font><font color="4444FF">)</font><font color="4444FF">)</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#2040a0">NULL</font><font color="4444FF" [...]
-<a name="line711">711</a>          <font color="4444FF"><strong>{</strong></font>
-<a name="line712">712</a>             <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Error: can't open output file %s<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">filename</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line713">713</a>             <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line714">714</a>             <font color="#2040a0">exit</font><font color="4444FF">(</font><font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line715">715</a>          <font color="4444FF"><strong>}</strong></font>
-<a name="line716">716</a> 
-<a name="line717">717</a>          <font color="#444444">/* save solution with global unknown numbers */</font>
-<a name="line718">718</a>          <font color="#2040a0">k</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font>
-<a name="line719">719</a>          <strong>for</strong> <font color="4444FF">(</font><font color="#2040a0">j</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font> <font color="#2040a0">j</font> <font color="4444FF"><</font> <font color="#2040a0">n</font><font color="4444FF">;</font> <font color="#2040a0">j</font><font color="4444FF">+</font><font color="4444FF">+</font><font color="4444FF">)</font>
-<a name="line720">720</a>             <strong>for</strong> <font color="4444FF">(</font><font color="#2040a0">i</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font> <font color="#2040a0">i</font> <font color="4444FF"><</font> <font color="#2040a0">n</font><font color="4444FF">;</font> <font color="#2040a0">i</font><font color="4444FF">+</font><font color="4444FF">+</font><font color="4444FF">)</font>
-<a name="line721">721</a>                <font color="#2040a0">fprintf</font><font color="4444FF">(</font><font color="#2040a0">file</font>, <font color="#008000">"%06d %.14e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">pj</font><font color="4444FF">*</font><font color="#2040a0">N</font><font color="4444FF">*</font><font color="#2040a0">n</font><font color="4444FF">*</font><font color="#2040a0">n</font><font color="4444FF">+</font><font color="#2040a0">pi</font [...]
-<a name="line722">722</a> 
-<a name="line723">723</a>          <font color="#2040a0">fflush</font><font color="4444FF">(</font><font color="#2040a0">file</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line724">724</a>          <font color="#2040a0">fclose</font><font color="4444FF">(</font><font color="#2040a0">file</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line725">725</a> 
-<a name="line726">726</a>          <font color="#444444">/* save local solution for variable v */</font>
-<a name="line727">727</a>          <font color="#2040a0">var</font> <font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">;</font>
-<a name="line728">728</a>          <font color="#2040a0">HYPRE_SStructVectorGetBoxValues</font><font color="4444FF">(</font><font color="#2040a0">x</font>, <font color="#2040a0">part</font>, <font color="#2040a0">ilower</font>, <font color="#2040a0">iupper</font>,
-<a name="line729">729</a>                                          <font color="#2040a0">var</font>, <font color="#2040a0">values</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line730">730</a> 
-<a name="line731">731</a>          <font color="#2040a0">sprintf</font><font color="4444FF">(</font><font color="#2040a0">filename</font>, <font color="#008000">"%s.%06d"</font>, <font color="#008000">"vis/ex9-v.sol"</font>, <font color="#2040a0">myid</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line732">732</a>          <strong>if</strong> <font color="4444FF">(</font><font color="4444FF">(</font><font color="#2040a0">file</font> <font color="4444FF">=</font> <font color="#2040a0">fopen</font><font color="4444FF">(</font><font color="#2040a0">filename</font>, <font color="#008000">"w"</font><font color="4444FF">)</font><font color="4444FF">)</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#2040a0">NULL</font><font color="4444FF" [...]
-<a name="line733">733</a>          <font color="4444FF"><strong>{</strong></font>
-<a name="line734">734</a>             <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Error: can't open output file %s<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">filename</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line735">735</a>             <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line736">736</a>             <font color="#2040a0">exit</font><font color="4444FF">(</font><font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line737">737</a>          <font color="4444FF"><strong>}</strong></font>
-<a name="line738">738</a> 
-<a name="line739">739</a>          <font color="#444444">/* save solution with global unknown numbers */</font>
-<a name="line740">740</a>          <font color="#2040a0">k</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font>
-<a name="line741">741</a>          <strong>for</strong> <font color="4444FF">(</font><font color="#2040a0">j</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font> <font color="#2040a0">j</font> <font color="4444FF"><</font> <font color="#2040a0">n</font><font color="4444FF">;</font> <font color="#2040a0">j</font><font color="4444FF">+</font><font color="4444FF">+</font><font color="4444FF">)</font>
-<a name="line742">742</a>             <strong>for</strong> <font color="4444FF">(</font><font color="#2040a0">i</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font> <font color="#2040a0">i</font> <font color="4444FF"><</font> <font color="#2040a0">n</font><font color="4444FF">;</font> <font color="#2040a0">i</font><font color="4444FF">+</font><font color="4444FF">+</font><font color="4444FF">)</font>
-<a name="line743">743</a>                <font color="#2040a0">fprintf</font><font color="4444FF">(</font><font color="#2040a0">file</font>, <font color="#008000">"%06d %.14e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">pj</font><font color="4444FF">*</font><font color="#2040a0">N</font><font color="4444FF">*</font><font color="#2040a0">n</font><font color="4444FF">*</font><font color="#2040a0">n</font><font color="4444FF">+</font><font color="#2040a0">pi</font [...]
-<a name="line744">744</a> 
-<a name="line745">745</a>          <font color="#2040a0">fflush</font><font color="4444FF">(</font><font color="#2040a0">file</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line746">746</a>          <font color="#2040a0">fclose</font><font color="4444FF">(</font><font color="#2040a0">file</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line747">747</a> 
-<a name="line748">748</a>          <font color="#2040a0">free</font><font color="4444FF">(</font><font color="#2040a0">values</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line632">632</a>          <font color="#2040a0">HYPRE_BoomerAMGSetOldDefault</font><font color="4444FF">(</font><font color="#2040a0">par_precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line633">633</a>          <font color="#2040a0">HYPRE_BoomerAMGSetStrongThreshold</font><font color="4444FF">(</font><font color="#2040a0">par_precond</font>, <font color="#FF0000">0.25</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line634">634</a>          <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">par_precond</font>, <font color="#FF0000">0.0</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line635">635</a>          <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">par_precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line636">636</a>          <font color="#2040a0">HYPRE_BoomerAMGSetPrintFileName</font><font color="4444FF">(</font><font color="#2040a0">par_precond</font>, <font color="#008000">"ex9.out.log"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line637">637</a>          <font color="#2040a0">HYPRE_BoomerAMGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">par_precond</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line638">638</a> 
+<a name="line639">639</a>          <font color="#444444">/* set the preconditioner */</font>
+<a name="line640">640</a>          <font color="#2040a0">HYPRE_ParCSRGMRESSetPrecond</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>,
+<a name="line641">641</a>                                      <font color="#2040a0">HYPRE_BoomerAMGSolve</font>,
+<a name="line642">642</a>                                      <font color="#2040a0">HYPRE_BoomerAMGSetup</font>,
+<a name="line643">643</a>                                      <font color="#2040a0">par_precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line644">644</a> 
+<a name="line645">645</a>          <font color="#444444">/* do the setup */</font>
+<a name="line646">646</a>          <font color="#2040a0">HYPRE_ParCSRGMRESSetup</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#2040a0">par_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line647">647</a> 
+<a name="line648">648</a>          <font color="#444444">/* do the solve */</font>
+<a name="line649">649</a>          <font color="#2040a0">HYPRE_ParCSRGMRESSolve</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#2040a0">par_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line650">650</a> 
+<a name="line651">651</a>          <font color="#444444">/* get some info */</font>
+<a name="line652">652</a>          <font color="#2040a0">HYPRE_GMRESGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="4444FF">&</font><font color="#2040a0">its</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line653">653</a>          <font color="#2040a0">HYPRE_GMRESGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>,
+<a name="line654">654</a>                                                  <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line655">655</a>          <font color="#444444">/* clean up */</font>
+<a name="line656">656</a>          <font color="#2040a0">HYPRE_ParCSRGMRESDestroy</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line657">657</a>          <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">par_precond</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line658">658</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line659">659</a>       <strong>else</strong> <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">solver_id</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">3</font><font color="4444FF">)</font> <font color="#444444">/* AMG */</font>
+<a name="line660">660</a>       <font color="4444FF"><strong>{</strong></font>
+<a name="line661">661</a>          <font color="#2040a0">HYPRE_BoomerAMGCreate</font><font color="4444FF">(</font><font color="4444FF">&</font><font color="#2040a0">par_solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line662">662</a>          <font color="#2040a0">HYPRE_BoomerAMGSetCoarsenType</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#FF0000">6</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line663">663</a>          <font color="#2040a0">HYPRE_BoomerAMGSetOldDefault</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line664">664</a>          <font color="#2040a0">HYPRE_BoomerAMGSetStrongThreshold</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#FF0000">0.25</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line665">665</a>          <font color="#2040a0">HYPRE_BoomerAMGSetTol</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#FF0000">1.9e-6</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line666">666</a>          <font color="#2040a0">HYPRE_BoomerAMGSetPrintLevel</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line667">667</a>          <font color="#2040a0">HYPRE_BoomerAMGSetPrintFileName</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#008000">"ex9.out.log"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line668">668</a>          <font color="#2040a0">HYPRE_BoomerAMGSetMaxIter</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#FF0000">50</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line669">669</a> 
+<a name="line670">670</a>          <font color="#444444">/* do the setup */</font>
+<a name="line671">671</a>          <font color="#2040a0">HYPRE_BoomerAMGSetup</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#2040a0">par_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line672">672</a> 
+<a name="line673">673</a>          <font color="#444444">/* do the solve */</font>
+<a name="line674">674</a>          <font color="#2040a0">HYPRE_BoomerAMGSolve</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="#2040a0">par_A</font>, <font color="#2040a0">par_b</font>, <font color="#2040a0">par_x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line675">675</a> 
+<a name="line676">676</a>          <font color="#444444">/* get some info */</font>
+<a name="line677">677</a>          <font color="#2040a0">HYPRE_BoomerAMGGetNumIterations</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>, <font color="4444FF">&</font><font color="#2040a0">its</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line678">678</a>          <font color="#2040a0">HYPRE_BoomerAMGGetFinalRelativeResidualNorm</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font>,
+<a name="line679">679</a>                                                      <font color="4444FF">&</font><font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line680">680</a>          <font color="#444444">/* clean up */</font>
+<a name="line681">681</a>          <font color="#2040a0">HYPRE_BoomerAMGDestroy</font><font color="4444FF">(</font><font color="#2040a0">par_solver</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line682">682</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line683">683</a>       <strong>else</strong>
+<a name="line684">684</a>       <font color="4444FF"><strong>{</strong></font>
+<a name="line685">685</a>          <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font><font color="#FF0000">0</font><font color="4444FF">)</font> <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font> ERROR: Invalid solver id specified.<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line686">686</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line687">687</a> 
+<a name="line688">688</a>       <font color="#444444">/* Gather the solution vector.  This needs to be done if:
+<a name="line689">689</a>          (1) the  object  type is parcsr OR
+<a name="line690">690</a>          (2) any one of the variables is NOT cell-centered */</font>
+<a name="line691">691</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">object_type</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#2040a0">HYPRE_PARCSR</font><font color="4444FF">)</font>
+<a name="line692">692</a>       <font color="4444FF"><strong>{</strong></font>
+<a name="line693">693</a>          <font color="#2040a0">HYPRE_SStructVectorGather</font><font color="4444FF">(</font><font color="#2040a0">x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line694">694</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line695">695</a> 
+<a name="line696">696</a>       <font color="#444444">/* Save the solution for GLVis visualization, see vis/glvis-ex7.sh */</font>
+<a name="line697">697</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">vis</font><font color="4444FF">)</font>
+<a name="line698">698</a>       <font color="4444FF"><strong>{</strong></font>
+<a name="line699">699</a>          <font color="#2040a0">FILE</font> <font color="4444FF">*</font><font color="#2040a0">file</font><font color="4444FF">;</font>
+<a name="line700">700</a>          <strong>char</strong> <font color="#2040a0">filename</font><font color="4444FF">[</font><font color="#FF0000">255</font><font color="4444FF">]</font><font color="4444FF">;</font>
+<a name="line701">701</a> 
+<a name="line702">702</a>          <strong>int</strong> <font color="#2040a0">k</font>, <font color="#2040a0">part</font> <font color="4444FF">=</font> <font color="#FF0000">0</font>, <font color="#2040a0">var</font><font color="4444FF">;</font>
+<a name="line703">703</a>          <strong>int</strong> <font color="#2040a0">nvalues</font> <font color="4444FF">=</font> <font color="#2040a0">n</font><font color="4444FF">*</font><font color="#2040a0">n</font><font color="4444FF">;</font>
+<a name="line704">704</a>          <strong>double</strong> <font color="4444FF">*</font><font color="#2040a0">values</font> <font color="4444FF">=</font> <font color="4444FF">(</font><strong>double</strong><font color="4444FF">*</font><font color="4444FF">)</font> <font color="#2040a0">calloc</font><font color="4444FF">(</font><font color="#2040a0">nvalues</font>, <strong>sizeof</strong><font color="4444FF">(</font><strong>double</strong><font color="4444FF">)</font><font color="4444FF"> [...]
+<a name="line705">705</a> 
+<a name="line706">706</a>          <font color="#444444">/* save local solution for variable u */</font>
+<a name="line707">707</a>          <font color="#2040a0">var</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font>
+<a name="line708">708</a>          <font color="#2040a0">HYPRE_SStructVectorGetBoxValues</font><font color="4444FF">(</font><font color="#2040a0">x</font>, <font color="#2040a0">part</font>, <font color="#2040a0">ilower</font>, <font color="#2040a0">iupper</font>,
+<a name="line709">709</a>                                          <font color="#2040a0">var</font>, <font color="#2040a0">values</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line710">710</a> 
+<a name="line711">711</a>          <font color="#2040a0">sprintf</font><font color="4444FF">(</font><font color="#2040a0">filename</font>, <font color="#008000">"%s.%06d"</font>, <font color="#008000">"vis/ex9-u.sol"</font>, <font color="#2040a0">myid</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line712">712</a>          <strong>if</strong> <font color="4444FF">(</font><font color="4444FF">(</font><font color="#2040a0">file</font> <font color="4444FF">=</font> <font color="#2040a0">fopen</font><font color="4444FF">(</font><font color="#2040a0">filename</font>, <font color="#008000">"w"</font><font color="4444FF">)</font><font color="4444FF">)</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#2040a0">NULL</font><font color="4444FF" [...]
+<a name="line713">713</a>          <font color="4444FF"><strong>{</strong></font>
+<a name="line714">714</a>             <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Error: can't open output file %s<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">filename</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line715">715</a>             <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line716">716</a>             <font color="#2040a0">exit</font><font color="4444FF">(</font><font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line717">717</a>          <font color="4444FF"><strong>}</strong></font>
+<a name="line718">718</a> 
+<a name="line719">719</a>          <font color="#444444">/* save solution with global unknown numbers */</font>
+<a name="line720">720</a>          <font color="#2040a0">k</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font>
+<a name="line721">721</a>          <strong>for</strong> <font color="4444FF">(</font><font color="#2040a0">j</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font> <font color="#2040a0">j</font> <font color="4444FF"><</font> <font color="#2040a0">n</font><font color="4444FF">;</font> <font color="#2040a0">j</font><font color="4444FF">+</font><font color="4444FF">+</font><font color="4444FF">)</font>
+<a name="line722">722</a>             <strong>for</strong> <font color="4444FF">(</font><font color="#2040a0">i</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font> <font color="#2040a0">i</font> <font color="4444FF"><</font> <font color="#2040a0">n</font><font color="4444FF">;</font> <font color="#2040a0">i</font><font color="4444FF">+</font><font color="4444FF">+</font><font color="4444FF">)</font>
+<a name="line723">723</a>                <font color="#2040a0">fprintf</font><font color="4444FF">(</font><font color="#2040a0">file</font>, <font color="#008000">"%06d %.14e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">pj</font><font color="4444FF">*</font><font color="#2040a0">N</font><font color="4444FF">*</font><font color="#2040a0">n</font><font color="4444FF">*</font><font color="#2040a0">n</font><font color="4444FF">+</font><font color="#2040a0">pi</font [...]
+<a name="line724">724</a> 
+<a name="line725">725</a>          <font color="#2040a0">fflush</font><font color="4444FF">(</font><font color="#2040a0">file</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line726">726</a>          <font color="#2040a0">fclose</font><font color="4444FF">(</font><font color="#2040a0">file</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line727">727</a> 
+<a name="line728">728</a>          <font color="#444444">/* save local solution for variable v */</font>
+<a name="line729">729</a>          <font color="#2040a0">var</font> <font color="4444FF">=</font> <font color="#FF0000">1</font><font color="4444FF">;</font>
+<a name="line730">730</a>          <font color="#2040a0">HYPRE_SStructVectorGetBoxValues</font><font color="4444FF">(</font><font color="#2040a0">x</font>, <font color="#2040a0">part</font>, <font color="#2040a0">ilower</font>, <font color="#2040a0">iupper</font>,
+<a name="line731">731</a>                                          <font color="#2040a0">var</font>, <font color="#2040a0">values</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line732">732</a> 
+<a name="line733">733</a>          <font color="#2040a0">sprintf</font><font color="4444FF">(</font><font color="#2040a0">filename</font>, <font color="#008000">"%s.%06d"</font>, <font color="#008000">"vis/ex9-v.sol"</font>, <font color="#2040a0">myid</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line734">734</a>          <strong>if</strong> <font color="4444FF">(</font><font color="4444FF">(</font><font color="#2040a0">file</font> <font color="4444FF">=</font> <font color="#2040a0">fopen</font><font color="4444FF">(</font><font color="#2040a0">filename</font>, <font color="#008000">"w"</font><font color="4444FF">)</font><font color="4444FF">)</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#2040a0">NULL</font><font color="4444FF" [...]
+<a name="line735">735</a>          <font color="4444FF"><strong>{</strong></font>
+<a name="line736">736</a>             <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Error: can't open output file %s<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">filename</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line737">737</a>             <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line738">738</a>             <font color="#2040a0">exit</font><font color="4444FF">(</font><font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line739">739</a>          <font color="4444FF"><strong>}</strong></font>
+<a name="line740">740</a> 
+<a name="line741">741</a>          <font color="#444444">/* save solution with global unknown numbers */</font>
+<a name="line742">742</a>          <font color="#2040a0">k</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font>
+<a name="line743">743</a>          <strong>for</strong> <font color="4444FF">(</font><font color="#2040a0">j</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font> <font color="#2040a0">j</font> <font color="4444FF"><</font> <font color="#2040a0">n</font><font color="4444FF">;</font> <font color="#2040a0">j</font><font color="4444FF">+</font><font color="4444FF">+</font><font color="4444FF">)</font>
+<a name="line744">744</a>             <strong>for</strong> <font color="4444FF">(</font><font color="#2040a0">i</font> <font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">;</font> <font color="#2040a0">i</font> <font color="4444FF"><</font> <font color="#2040a0">n</font><font color="4444FF">;</font> <font color="#2040a0">i</font><font color="4444FF">+</font><font color="4444FF">+</font><font color="4444FF">)</font>
+<a name="line745">745</a>                <font color="#2040a0">fprintf</font><font color="4444FF">(</font><font color="#2040a0">file</font>, <font color="#008000">"%06d %.14e<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">pj</font><font color="4444FF">*</font><font color="#2040a0">N</font><font color="4444FF">*</font><font color="#2040a0">n</font><font color="4444FF">*</font><font color="#2040a0">n</font><font color="4444FF">+</font><font color="#2040a0">pi</font [...]
+<a name="line746">746</a> 
+<a name="line747">747</a>          <font color="#2040a0">fflush</font><font color="4444FF">(</font><font color="#2040a0">file</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line748">748</a>          <font color="#2040a0">fclose</font><font color="4444FF">(</font><font color="#2040a0">file</font><font color="4444FF">)</font><font color="4444FF">;</font>
 <a name="line749">749</a> 
-<a name="line750">750</a>          <font color="#444444">/* save global finite element mesh */</font>
-<a name="line751">751</a>          <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
-<a name="line752">752</a>             <font color="#2040a0">GLVis_PrintGlobalSquareMesh</font><font color="4444FF">(</font><font color="#008000">"vis/ex9.mesh"</font>, <font color="#2040a0">N</font><font color="4444FF">*</font><font color="#2040a0">n</font><font color="4444FF">-</font><font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line753">753</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line754">754</a> 
-<a name="line755">755</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
-<a name="line756">756</a>       <font color="4444FF"><strong>{</strong></font>
-<a name="line757">757</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line758">758</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %d<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">its</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line759">759</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %g<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line760">760</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line761">761</a>       <font color="4444FF"><strong>}</strong></font>
-<a name="line762">762</a>    <font color="4444FF"><strong>}</strong></font>
-<a name="line763">763</a> 
-<a name="line764">764</a>    <font color="#444444">/* Free memory */</font>
-<a name="line765">765</a>    <font color="#2040a0">HYPRE_SStructGridDestroy</font><font color="4444FF">(</font><font color="#2040a0">grid</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line766">766</a>    <font color="#2040a0">HYPRE_SStructStencilDestroy</font><font color="4444FF">(</font><font color="#2040a0">stencil_v</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line767">767</a>    <font color="#2040a0">HYPRE_SStructStencilDestroy</font><font color="4444FF">(</font><font color="#2040a0">stencil_u</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line768">768</a>    <font color="#2040a0">HYPRE_SStructGraphDestroy</font><font color="4444FF">(</font><font color="#2040a0">graph</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line769">769</a>    <font color="#2040a0">HYPRE_SStructMatrixDestroy</font><font color="4444FF">(</font><font color="#2040a0">A</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line770">770</a>    <font color="#2040a0">HYPRE_SStructVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">b</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line771">771</a>    <font color="#2040a0">HYPRE_SStructVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">x</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line772">772</a> 
-<a name="line773">773</a>    <font color="#444444">/* Finalize MPI */</font>
-<a name="line774">774</a>    <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line775">775</a> 
-<a name="line776">776</a>    <strong>return</strong> <font color="4444FF">(</font><font color="#FF0000">0</font><font color="4444FF">)</font><font color="4444FF">;</font>
-<a name="line777">777</a> <font color="4444FF"><strong>}</strong></font>
+<a name="line750">750</a>          <font color="#2040a0">free</font><font color="4444FF">(</font><font color="#2040a0">values</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line751">751</a> 
+<a name="line752">752</a>          <font color="#444444">/* save global finite element mesh */</font>
+<a name="line753">753</a>          <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
+<a name="line754">754</a>             <font color="#2040a0">GLVis_PrintGlobalSquareMesh</font><font color="4444FF">(</font><font color="#008000">"vis/ex9.mesh"</font>, <font color="#2040a0">N</font><font color="4444FF">*</font><font color="#2040a0">n</font><font color="4444FF">-</font><font color="#FF0000">1</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line755">755</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line756">756</a> 
+<a name="line757">757</a>       <strong>if</strong> <font color="4444FF">(</font><font color="#2040a0">myid</font> <font color="4444FF">=</font><font color="4444FF">=</font> <font color="#FF0000">0</font><font color="4444FF">)</font>
+<a name="line758">758</a>       <font color="4444FF"><strong>{</strong></font>
+<a name="line759">759</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line760">760</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Iterations = %d<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">its</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line761">761</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"Final Relative Residual Norm = %g<font color="#77dd77">\n</font>"</font>, <font color="#2040a0">final_res_norm</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line762">762</a>          <font color="#2040a0">printf</font><font color="4444FF">(</font><font color="#008000">"<font color="#77dd77">\n</font>"</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line763">763</a>       <font color="4444FF"><strong>}</strong></font>
+<a name="line764">764</a>    <font color="4444FF"><strong>}</strong></font>
+<a name="line765">765</a> 
+<a name="line766">766</a>    <font color="#444444">/* Free memory */</font>
+<a name="line767">767</a>    <font color="#2040a0">HYPRE_SStructGridDestroy</font><font color="4444FF">(</font><font color="#2040a0">grid</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line768">768</a>    <font color="#2040a0">HYPRE_SStructStencilDestroy</font><font color="4444FF">(</font><font color="#2040a0">stencil_v</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line769">769</a>    <font color="#2040a0">HYPRE_SStructStencilDestroy</font><font color="4444FF">(</font><font color="#2040a0">stencil_u</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line770">770</a>    <font color="#2040a0">HYPRE_SStructGraphDestroy</font><font color="4444FF">(</font><font color="#2040a0">graph</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line771">771</a>    <font color="#2040a0">HYPRE_SStructMatrixDestroy</font><font color="4444FF">(</font><font color="#2040a0">A</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line772">772</a>    <font color="#2040a0">HYPRE_SStructVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">b</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line773">773</a>    <font color="#2040a0">HYPRE_SStructVectorDestroy</font><font color="4444FF">(</font><font color="#2040a0">x</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line774">774</a> 
+<a name="line775">775</a>    <font color="#444444">/* Finalize MPI */</font>
+<a name="line776">776</a>    <font color="#2040a0">MPI_Finalize</font><font color="4444FF">(</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line777">777</a> 
+<a name="line778">778</a>    <strong>return</strong> <font color="4444FF">(</font><font color="#FF0000">0</font><font color="4444FF">)</font><font color="4444FF">;</font>
+<a name="line779">779</a> <font color="4444FF"><strong>}</strong></font>
 
 </pre>
 <hr>
diff --git a/src/examples/ex10.cxx b/src/examples/ex10.cxx
index a471ccb..4e029ff 100644
--- a/src/examples/ex10.cxx
+++ b/src/examples/ex10.cxx
@@ -133,7 +133,7 @@ int main(int argc, char *argv[])
       paramStrings[0] = new char[100];
       strcpy(paramStrings[0], "externalSolver HYPRE");
       feiPtr->parameters(1, paramStrings);
-      delete paramStrings[0];
+      delete [] paramStrings[0];
       delete [] paramStrings;
    }
 
@@ -449,7 +449,7 @@ int main(int argc, char *argv[])
       strcpy(paramStrings[4], "tolerance 1e-6");
       strcpy(paramStrings[5], "gmresDim 30");
       strcpy(paramStrings[6], "amgNumSweeps 1");
-      strcpy(paramStrings[7], "amgCoarsenType falgout");
+      strcpy(paramStrings[7], "amgCoarsenType hmis");
       strcpy(paramStrings[8], "amgRelaxType hybridsym");
       strcpy(paramStrings[9], "amgSystemSize 1");
       strcpy(paramStrings[10], "amgStrongThreshold 0.25");
diff --git a/src/examples/ex11.c b/src/examples/ex11.c
index 91a2dad..8b7edcf 100644
--- a/src/examples/ex11.c
+++ b/src/examples/ex11.c
@@ -232,9 +232,7 @@ int main (int argc, char *argv[])
    {
       HYPRE_BoomerAMGCreate(&precond);
       HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info */
-      HYPRE_BoomerAMGSetCoarsenType(precond, 6);
-      HYPRE_BoomerAMGSetRelaxType(precond, 6); /* Sym G.S./Jacobi hybrid */
-      HYPRE_BoomerAMGSetNumSweeps(precond, 1);
+      HYPRE_BoomerAMGSetNumSweeps(precond, 2); /* 2 sweeps of smoothing */
       HYPRE_BoomerAMGSetTol(precond, 0.0); /* conv. tolerance zero */
       HYPRE_BoomerAMGSetMaxIter(precond, 1); /* do only one iteration! */
    }
diff --git a/src/examples/ex12.c b/src/examples/ex12.c
index 2de2aab..ca41094 100644
--- a/src/examples/ex12.c
+++ b/src/examples/ex12.c
@@ -449,7 +449,6 @@ int main (int argc, char *argv[])
       HYPRE_BoomerAMGSetMaxIter(precond, 1);
       HYPRE_BoomerAMGSetTol(precond, 0.0);
       HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info */
-      HYPRE_BoomerAMGSetCoarsenType(precond, 6);
       HYPRE_BoomerAMGSetRelaxType(precond, 6); /* Sym G.S./Jacobi hybrid */
       HYPRE_BoomerAMGSetNumSweeps(precond, 1);
 
diff --git a/src/examples/ex12f.f b/src/examples/ex12f.f
index c69bb22..f2240a9 100644
--- a/src/examples/ex12f.f
+++ b/src/examples/ex12f.f
@@ -449,6 +449,7 @@ c        Set BoomerAMG parameters
 c        Print amg solution info
          call HYPRE_BoomerAMGSetPrintLevel(precond, 1, ierr)
          call HYPRE_BoomerAMGSetCoarsenType(precond, 6, ierr)
+         call HYPRE_BoomerAMGSetOldDefault(precond, ierr)
 c        Sym G.S./Jacobi hybrid
          call HYPRE_BoomerAMGSetRelaxType(precond, 6, ierr)
          call HYPRE_BoomerAMGSetNumSweeps(precond, 1, ierr)
diff --git a/src/examples/ex13.c b/src/examples/ex13.c
index c718b88..f62a21d 100644
--- a/src/examples/ex13.c
+++ b/src/examples/ex13.c
@@ -593,7 +593,7 @@ int main (int argc, char *argv[])
       /* Here we construct a BoomerAMG solver.  See the other SStruct examples
          as well as the Reference manual for additional solver choices. */
       HYPRE_BoomerAMGCreate(&solver);
-      HYPRE_BoomerAMGSetCoarsenType(solver, 6);
+      HYPRE_BoomerAMGSetOldDefault(solver);
       HYPRE_BoomerAMGSetStrongThreshold(solver, 0.25);
       HYPRE_BoomerAMGSetTol(solver, 1e-6);
       HYPRE_BoomerAMGSetPrintLevel(solver, 2);
diff --git a/src/examples/ex14.c b/src/examples/ex14.c
index c133d05..a965053 100644
--- a/src/examples/ex14.c
+++ b/src/examples/ex14.c
@@ -551,7 +551,7 @@ int main (int argc, char *argv[])
       /* Here we construct a BoomerAMG solver.  See the other SStruct examples
          as well as the Reference manual for additional solver choices. */
       HYPRE_BoomerAMGCreate(&solver);
-      HYPRE_BoomerAMGSetCoarsenType(solver, 6);
+      HYPRE_BoomerAMGSetOldDefault(solver);
       HYPRE_BoomerAMGSetStrongThreshold(solver, 0.25);
       HYPRE_BoomerAMGSetTol(solver, 1e-6);
       HYPRE_BoomerAMGSetPrintLevel(solver, 2);
diff --git a/src/examples/ex5.c b/src/examples/ex5.c
index 2522d94..068555f 100644
--- a/src/examples/ex5.c
+++ b/src/examples/ex5.c
@@ -309,8 +309,9 @@ int main (int argc, char *argv[])
 
       /* Set some parameters (See Reference Manual for more parameters) */
       HYPRE_BoomerAMGSetPrintLevel(solver, 3);  /* print solve info + parameters */
-      HYPRE_BoomerAMGSetCoarsenType(solver, 6); /* Falgout coarsening */
+      HYPRE_BoomerAMGSetOldDefault(solver); /* Falgout coarsening with modified classical interpolaiton */
       HYPRE_BoomerAMGSetRelaxType(solver, 3);   /* G-S/Jacobi hybrid relaxation */
+      HYPRE_BoomerAMGSetRelaxOrder(solver, 1);   /* uses C/F relaxation */
       HYPRE_BoomerAMGSetNumSweeps(solver, 1);   /* Sweeeps on each level */
       HYPRE_BoomerAMGSetMaxLevels(solver, 20);  /* maximum number of levels */
       HYPRE_BoomerAMGSetTol(solver, 1e-7);      /* conv. tolerance */
@@ -387,6 +388,7 @@ int main (int argc, char *argv[])
       HYPRE_BoomerAMGCreate(&precond);
       HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info */
       HYPRE_BoomerAMGSetCoarsenType(precond, 6);
+      HYPRE_BoomerAMGSetOldDefault(precond); 
       HYPRE_BoomerAMGSetRelaxType(precond, 6); /* Sym G.S./Jacobi hybrid */
       HYPRE_BoomerAMGSetNumSweeps(precond, 1);
       HYPRE_BoomerAMGSetTol(precond, 0.0); /* conv. tolerance zero */
@@ -493,6 +495,7 @@ int main (int argc, char *argv[])
       HYPRE_BoomerAMGCreate(&precond);
       HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info */
       HYPRE_BoomerAMGSetCoarsenType(precond, 6);
+      HYPRE_BoomerAMGSetOldDefault(precond);
       HYPRE_BoomerAMGSetRelaxType(precond, 6); /* Sym G.S./Jacobi hybrid */
       HYPRE_BoomerAMGSetNumSweeps(precond, 1);
       HYPRE_BoomerAMGSetTol(precond, 0.0); /* conv. tolerance zero */
diff --git a/src/examples/ex5big.c b/src/examples/ex5big.c
index 6a8d104..8a67c7c 100644
--- a/src/examples/ex5big.c
+++ b/src/examples/ex5big.c
@@ -308,8 +308,9 @@ int main (int argc, char *argv[])
 
       /* Set some parameters (See Reference Manual for more parameters) */
       HYPRE_BoomerAMGSetPrintLevel(solver, 3);  /* print solve info + parameters */
-      HYPRE_BoomerAMGSetCoarsenType(solver, 6); /* Falgout coarsening */
+      HYPRE_BoomerAMGSetOldDefault(solver); /* Falgout coarsening with modified classical interpolation */
       HYPRE_BoomerAMGSetRelaxType(solver, 3);   /* G-S/Jacobi hybrid relaxation */
+      HYPRE_BoomerAMGSetRelaxOrder(solver, 1);   /* Uses C/F relaxation */
       HYPRE_BoomerAMGSetNumSweeps(solver, 1);   /* Sweeeps on each level */
       HYPRE_BoomerAMGSetMaxLevels(solver, 20);  /* maximum number of levels */
       HYPRE_BoomerAMGSetTol(solver, 1e-7);      /* conv. tolerance */
@@ -386,6 +387,7 @@ int main (int argc, char *argv[])
       HYPRE_BoomerAMGCreate(&precond);
       HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info */
       HYPRE_BoomerAMGSetCoarsenType(precond, 6);
+      HYPRE_BoomerAMGSetOldDefault(precond);
       HYPRE_BoomerAMGSetRelaxType(precond, 6); /* Sym G.S./Jacobi hybrid */
       HYPRE_BoomerAMGSetNumSweeps(precond, 1);
       HYPRE_BoomerAMGSetTol(precond, 0.0); /* conv. tolerance zero */
@@ -492,6 +494,7 @@ int main (int argc, char *argv[])
       HYPRE_BoomerAMGCreate(&precond);
       HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info */
       HYPRE_BoomerAMGSetCoarsenType(precond, 6);
+      HYPRE_BoomerAMGSetOldDefault(precond);
       HYPRE_BoomerAMGSetRelaxType(precond, 6); /* Sym G.S./Jacobi hybrid */
       HYPRE_BoomerAMGSetNumSweeps(precond, 1);
       HYPRE_BoomerAMGSetTol(precond, 0.0); /* conv. tolerance zero */
diff --git a/src/examples/ex5f.f b/src/examples/ex5f.f
index 9a757fd..8fbd54a 100644
--- a/src/examples/ex5f.f
+++ b/src/examples/ex5f.f
@@ -230,10 +230,12 @@ c        Set some parameters (See Reference Manual for more parameters)
 
 c        print solve info + parameters 
          call HYPRE_BoomerAMGSetPrintLevel(solver, 3, ierr)  
-c        Falgout coarsening
-         call HYPRE_BoomerAMGSetCoarsenType(solver, 6, ierr) 
+c        old defaults, Falgout coarsening, mod. class. interpolation
+         call HYPRE_BoomerAMGSetOldDefault(solver, ierr) 
 c        G-S/Jacobi hybrid relaxation 
          call HYPRE_BoomerAMGSetRelaxType(solver, 3, ierr)     
+c        C/F relaxation 
+         call HYPRE_BoomerAMGSetRelaxOrder(solver, 1, ierr)     
 c        Sweeeps on each level
          call HYPRE_BoomerAMGSetNumSweeps(solver, 1, ierr)  
 c         maximum number of levels 
@@ -336,6 +338,8 @@ c        print less solver info since a preconditioner
          call HYPRE_BoomerAMGSetPrintLevel(precond, 1, ierr); 
 c        Falgout coarsening
          call HYPRE_BoomerAMGSetCoarsenType(precond, 6, ierr) 
+c        old defaults
+         call HYPRE_BoomerAMGSetOldDefault(precond, ierr) 
 c        SYMMETRIC G-S/Jacobi hybrid relaxation 
          call HYPRE_BoomerAMGSetRelaxType(precond, 6, ierr)     
 c        Sweeeps on each level
diff --git a/src/examples/ex9.c b/src/examples/ex9.c
index 339f2af..1058e4a 100644
--- a/src/examples/ex9.c
+++ b/src/examples/ex9.c
@@ -629,6 +629,7 @@ int main (int argc, char *argv[])
          /* use BoomerAMG as preconditioner */
          HYPRE_BoomerAMGCreate(&par_precond);
          HYPRE_BoomerAMGSetCoarsenType(par_precond, 6);
+         HYPRE_BoomerAMGSetOldDefault(par_precond);
          HYPRE_BoomerAMGSetStrongThreshold(par_precond, 0.25);
          HYPRE_BoomerAMGSetTol(par_precond, 0.0);
          HYPRE_BoomerAMGSetPrintLevel(par_precond, 1);
@@ -659,6 +660,7 @@ int main (int argc, char *argv[])
       {
          HYPRE_BoomerAMGCreate(&par_solver);
          HYPRE_BoomerAMGSetCoarsenType(par_solver, 6);
+         HYPRE_BoomerAMGSetOldDefault(par_solver);
          HYPRE_BoomerAMGSetStrongThreshold(par_solver, 0.25);
          HYPRE_BoomerAMGSetTol(par_solver, 1.9e-6);
          HYPRE_BoomerAMGSetPrintLevel(par_solver, 1);
diff --git a/src/krylov/lgmres.c b/src/krylov/lgmres.c
index b82b951..fdbbf00 100644
--- a/src/krylov/lgmres.c
+++ b/src/krylov/lgmres.c
@@ -245,9 +245,12 @@ hypre_LGMRESSetup( void *lgmres_vdata,
    }
  
    /* lgmres mod */
-   (lgmres_data -> aug_vecs) = (void**)(*(lgmres_functions->CreateVectorArray))(aug_dim+1,x); /* one extra */
-   (lgmres_data -> a_aug_vecs) = (void**)(*(lgmres_functions->CreateVectorArray))(aug_dim,x);
-   (lgmres_data -> aug_order) = hypre_CTAllocF(HYPRE_Int,aug_dim,lgmres_functions); 
+   if ((lgmres_data -> aug_vecs) == NULL)
+      (lgmres_data -> aug_vecs) = (void**)(*(lgmres_functions->CreateVectorArray))(aug_dim+1,x); /* one extra */
+   if ((lgmres_data -> a_aug_vecs) == NULL)
+      (lgmres_data -> a_aug_vecs) = (void**)(*(lgmres_functions->CreateVectorArray))(aug_dim,x);
+   if ((lgmres_data -> aug_order) == NULL)
+      (lgmres_data -> aug_order) = hypre_CTAllocF(HYPRE_Int,aug_dim,lgmres_functions); 
    /*---*/
 
 
diff --git a/src/parcsr_block_mv/par_block_nodal_systems.c b/src/parcsr_block_mv/par_block_nodal_systems.c
index e96805c..20d6ba6 100644
--- a/src/parcsr_block_mv/par_block_nodal_systems.c
+++ b/src/parcsr_block_mv/par_block_nodal_systems.c
@@ -106,7 +106,7 @@ hypre_BoomerAMGBlockCreateNodalA(hypre_ParCSRBlockMatrix *A,
       comm_pkg = hypre_ParCSRBlockMatrixCommPkg(A);
    }
 
-   norm_type = abs(option);
+   norm_type = hypre_abs(option);
 
 
 /* Set up the new matrix AN */
diff --git a/src/parcsr_ls/F90_HYPRE_parcsr_amg.c b/src/parcsr_ls/F90_HYPRE_parcsr_amg.c
index 6e534d5..3ba55a9 100644
--- a/src/parcsr_ls/F90_HYPRE_parcsr_amg.c
+++ b/src/parcsr_ls/F90_HYPRE_parcsr_amg.c
@@ -531,6 +531,20 @@ hypre_F90_IFACE(hypre_boomeramggetmeasuretype, HYPRE_BOOMERAMGGETMEASURETYPE)
 }
 
 /*--------------------------------------------------------------------------
+ * HYPRE_BoomerAMGSetOldDefault
+ *--------------------------------------------------------------------------*/
+
+void
+hypre_F90_IFACE(hypre_boomeramgsetolddefault, HYPRE_BOOMERAMGSETOLDDEFAULT)
+   ( hypre_F90_Obj *solver,
+     hypre_F90_Int *ierr          )
+{
+   *ierr = (hypre_F90_Int)
+      ( HYPRE_BoomerAMGSetOldDefault(
+           hypre_F90_PassObj (HYPRE_Solver, solver) ) );
+}
+
+/*--------------------------------------------------------------------------
  * HYPRE_BoomerAMGSetSetupType
  *--------------------------------------------------------------------------*/
 
@@ -1841,6 +1855,38 @@ hypre_F90_IFACE(hypre_boomeramgsetmultaddpmx, HYPRE_BOOMERAMGSETMULTADDPMX)
 }
 
 /*--------------------------------------------------------------------------
+ * HYPRE_BoomerAMGSetAddRelaxType
+ *--------------------------------------------------------------------------*/
+
+void
+hypre_F90_IFACE(hypre_boomeramgsetaddrlxtype, HYPRE_BOOMERAMGSETADDRLXTYPE)
+   ( hypre_F90_Obj *solver,
+     hypre_F90_Int *add_rlx_type,
+     hypre_F90_Int *ierr          )
+{
+   *ierr = (hypre_F90_Int)
+      ( HYPRE_BoomerAMGSetAddRelaxType(
+           hypre_F90_PassObj (HYPRE_Solver, solver),
+           hypre_F90_PassInt (add_rlx_type) ) );
+}
+
+/*--------------------------------------------------------------------------
+ * HYPRE_BoomerAMGSetAddRelaxWt
+ *--------------------------------------------------------------------------*/
+
+void
+hypre_F90_IFACE(hypre_boomeramgsetaddrlxwt, HYPRE_BOOMERAMGSETADDRLXWT)
+   ( hypre_F90_Obj *solver,
+     hypre_F90_Real *add_rlx_wt,
+     hypre_F90_Int *ierr          )
+{
+   *ierr = (hypre_F90_Int)
+      ( HYPRE_BoomerAMGSetAddRelaxWt(
+           hypre_F90_PassObj (HYPRE_Solver, solver),
+           hypre_F90_PassReal (add_rlx_wt) ) );
+}
+
+/*--------------------------------------------------------------------------
  * HYPRE_BoomerAMGSetSeqThreshold
  *--------------------------------------------------------------------------*/
 
diff --git a/src/parcsr_ls/HYPRE_parcsr_amg.c b/src/parcsr_ls/HYPRE_parcsr_amg.c
index e78f238..2a4dbd7 100644
--- a/src/parcsr_ls/HYPRE_parcsr_amg.c
+++ b/src/parcsr_ls/HYPRE_parcsr_amg.c
@@ -415,6 +415,19 @@ HYPRE_BoomerAMGGetMeasureType( HYPRE_Solver solver,
 }
 
 /*--------------------------------------------------------------------------
+ * HYPRE_BoomerAMGSetOldDefault
+ *--------------------------------------------------------------------------*/
+
+HYPRE_Int
+HYPRE_BoomerAMGSetOldDefault( HYPRE_Solver solver)
+{
+   HYPRE_BoomerAMGSetCoarsenType( solver, 6 );
+   HYPRE_BoomerAMGSetInterpType( solver, 0 );
+   HYPRE_BoomerAMGSetPMaxElmts( solver, 0 );
+   return hypre_error_flag;
+}
+
+/*--------------------------------------------------------------------------
  * HYPRE_BoomerAMGSetSetupType
  *--------------------------------------------------------------------------*/
 
@@ -1231,6 +1244,27 @@ HYPRE_BoomerAMGSetMultAddTruncFactor( HYPRE_Solver  solver,
 }
 
 /*--------------------------------------------------------------------------
+ * HYPRE_BoomerAMGSetAddRelaxWt
+ *--------------------------------------------------------------------------*/
+
+HYPRE_Int
+HYPRE_BoomerAMGSetAddRelaxWt( HYPRE_Solver  solver,
+                                  HYPRE_Real        add_rlx_wt  )
+{
+   return( hypre_BoomerAMGSetAddRelaxWt( (void *) solver, add_rlx_wt ) );
+}
+
+/*--------------------------------------------------------------------------
+ * HYPRE_BoomerAMGSetAddRelaxType
+ *--------------------------------------------------------------------------*/
+
+HYPRE_Int
+HYPRE_BoomerAMGSetAddRelaxType( HYPRE_Solver  solver,
+                                  HYPRE_Int        add_rlx_type  )
+{
+   return( hypre_BoomerAMGSetAddRelaxType( (void *) solver, add_rlx_type ) );
+}
+/*--------------------------------------------------------------------------
  * HYPRE_BoomerAMGSetAggP12TruncFactor
  *--------------------------------------------------------------------------*/
 
diff --git a/src/parcsr_ls/HYPRE_parcsr_ls.h b/src/parcsr_ls/HYPRE_parcsr_ls.h
index 6292ff3..c5e35c2 100644
--- a/src/parcsr_ls/HYPRE_parcsr_ls.h
+++ b/src/parcsr_ls/HYPRE_parcsr_ls.h
@@ -139,6 +139,13 @@ HYPRE_Int HYPRE_BoomerAMGSolveT(HYPRE_Solver       solver,
                                 HYPRE_ParVector    x);
 
 /**
+ * Recovers old default for coarsening and interpolation, i.e Falgout 
+ * coarsening and untruncated modified classical interpolation.
+ * This option might be preferred for 2 dimensional problems.
+ **/
+HYPRE_Int HYPRE_BoomerAMGSetOldDefault(HYPRE_Solver       solver);
+
+/**
  * Returns the residual.
  **/
 HYPRE_Int HYPRE_BoomerAMGGetResidual(HYPRE_Solver     solver,
@@ -589,6 +596,24 @@ HYPRE_Int HYPRE_BoomerAMGSetMultAddTruncFactor(HYPRE_Solver solver,
  **/
 HYPRE_Int HYPRE_BoomerAMGSetMultAddPMaxElmts(HYPRE_Solver solver,
                                          HYPRE_Int    add_P_max_elmts);
+/**
+ * (Optional) Defines the relaxation type used in the (mult)additive cycle
+ * portion (also affects simple method.) 
+ * The default is 18 (L1-Jacobi).
+ * Currently the only other option allowed is 0 (Jacobi) which should be 
+ * used in combination with HYPRE_BoomerAMGSetAddRelaxWt.
+ **/
+HYPRE_Int HYPRE_BoomerAMGSetAddRelaxType(HYPRE_Solver solver,
+                                         HYPRE_Int    add_rlx_type);
+
+/**
+ * (Optional) Defines the relaxation weight used for Jacobi within the 
+ * (mult)additive or simple cycle portion.
+ * The default is 1. 
+ * The weight only affects the Jacobi method, and has no effect on L1-Jacobi
+ **/
+HYPRE_Int HYPRE_BoomerAMGSetAddRelaxWt(HYPRE_Solver solver,
+                                         HYPRE_Real    add_rlx_wt);
 
 /**
  * (Optional) Sets maximal size for agglomeration or redundant coarse grid solve. 
diff --git a/src/parcsr_ls/_hypre_parcsr_ls.h b/src/parcsr_ls/_hypre_parcsr_ls.h
index 0cf5d12..22807d1 100644
--- a/src/parcsr_ls/_hypre_parcsr_ls.h
+++ b/src/parcsr_ls/_hypre_parcsr_ls.h
@@ -241,7 +241,9 @@ typedef struct
    HYPRE_Int      mult_additive;
    HYPRE_Int      simple;
    HYPRE_Int      add_P_max_elmts;
-   HYPRE_Int      add_trunc_factor;
+   HYPRE_Real     add_trunc_factor;
+   HYPRE_Int      add_rlx_type;
+   HYPRE_Real     add_rlx_wt;
    hypre_ParCSRMatrix *Lambda;
    hypre_ParVector *Rtilde;
    hypre_ParVector *Xtilde;
@@ -437,6 +439,8 @@ typedef struct
 #define hypre_ParAMGDataSimple(amg_data) ((amg_data)->simple)
 #define hypre_ParAMGDataMultAddPMaxElmts(amg_data) ((amg_data)->add_P_max_elmts)
 #define hypre_ParAMGDataMultAddTruncFactor(amg_data) ((amg_data)->add_trunc_factor)
+#define hypre_ParAMGDataAddRelaxType(amg_data) ((amg_data)->add_rlx_type)
+#define hypre_ParAMGDataAddRelaxWt(amg_data) ((amg_data)->add_rlx_wt)
 #define hypre_ParAMGDataLambda(amg_data) ((amg_data)->Lambda)
 #define hypre_ParAMGDataRtilde(amg_data) ((amg_data)->Rtilde)
 #define hypre_ParAMGDataXtilde(amg_data) ((amg_data)->Xtilde)
@@ -744,6 +748,7 @@ HYPRE_Int HYPRE_BoomerAMGGetCoarsenType ( HYPRE_Solver solver , HYPRE_Int *coars
 HYPRE_Int HYPRE_BoomerAMGSetMeasureType ( HYPRE_Solver solver , HYPRE_Int measure_type );
 HYPRE_Int HYPRE_BoomerAMGGetMeasureType ( HYPRE_Solver solver , HYPRE_Int *measure_type );
 HYPRE_Int HYPRE_BoomerAMGSetSetupType ( HYPRE_Solver solver , HYPRE_Int setup_type );
+HYPRE_Int HYPRE_BoomerAMGSetOldDefault ( HYPRE_Solver solver );
 HYPRE_Int HYPRE_BoomerAMGSetCycleType ( HYPRE_Solver solver , HYPRE_Int cycle_type );
 HYPRE_Int HYPRE_BoomerAMGGetCycleType ( HYPRE_Solver solver , HYPRE_Int *cycle_type );
 HYPRE_Int HYPRE_BoomerAMGSetTol ( HYPRE_Solver solver , HYPRE_Real tol );
@@ -817,6 +822,8 @@ HYPRE_Int HYPRE_BoomerAMGSetAggP12TruncFactor ( HYPRE_Solver solver , HYPRE_Real
 HYPRE_Int HYPRE_BoomerAMGSetAggPMaxElmts ( HYPRE_Solver solver , HYPRE_Int agg_P_max_elmts );
 HYPRE_Int HYPRE_BoomerAMGSetAddPMaxElmts ( HYPRE_Solver solver , HYPRE_Int add_P_max_elmts );
 HYPRE_Int HYPRE_BoomerAMGSetMultAddPMaxElmts ( HYPRE_Solver solver , HYPRE_Int add_P_max_elmts );
+HYPRE_Int HYPRE_BoomerAMGSetAddRelaxType ( HYPRE_Solver solver , HYPRE_Int add_rlx_type );
+HYPRE_Int HYPRE_BoomerAMGSetAddRelaxWt ( HYPRE_Solver solver , HYPRE_Real add_rlx_wt );
 HYPRE_Int HYPRE_BoomerAMGSetAggP12MaxElmts ( HYPRE_Solver solver , HYPRE_Int agg_P12_max_elmts );
 HYPRE_Int HYPRE_BoomerAMGSetNumCRRelaxSteps ( HYPRE_Solver solver , HYPRE_Int num_CR_relax_steps );
 HYPRE_Int HYPRE_BoomerAMGSetCRRate ( HYPRE_Solver solver , HYPRE_Real CR_rate );
@@ -1214,6 +1221,8 @@ HYPRE_Int hypre_BoomerAMGSetAggNumLevels ( void *data , HYPRE_Int agg_num_levels
 HYPRE_Int hypre_BoomerAMGSetAggInterpType ( void *data , HYPRE_Int agg_interp_type );
 HYPRE_Int hypre_BoomerAMGSetAggPMaxElmts ( void *data , HYPRE_Int agg_P_max_elmts );
 HYPRE_Int hypre_BoomerAMGSetMultAddPMaxElmts ( void *data , HYPRE_Int add_P_max_elmts );
+HYPRE_Int hypre_BoomerAMGSetAddRelaxType ( void *data , HYPRE_Int add_rlx_type );
+HYPRE_Int hypre_BoomerAMGSetAddRelaxWt ( void *data , HYPRE_Real add_rlx_wt );
 HYPRE_Int hypre_BoomerAMGSetAggP12MaxElmts ( void *data , HYPRE_Int agg_P12_max_elmts );
 HYPRE_Int hypre_BoomerAMGSetAggTruncFactor ( void *data , HYPRE_Real agg_trunc_factor );
 HYPRE_Int hypre_BoomerAMGSetMultAddTruncFactor ( void *data , HYPRE_Real add_trunc_factor );
diff --git a/src/parcsr_ls/amg_hybrid.c b/src/parcsr_ls/amg_hybrid.c
index 683d01e..4c52cb5 100644
--- a/src/parcsr_ls/amg_hybrid.c
+++ b/src/parcsr_ls/amg_hybrid.c
@@ -944,7 +944,7 @@ hypre_AMGHybridSetMinCoarseSize( void *AMGhybrid_vdata,
       hypre_error_in_arg(1);
       return hypre_error_flag;
    }
-   if (min_coarse_size < 1)
+   if (min_coarse_size < 0)
    {
       hypre_error_in_arg(2);
       return hypre_error_flag;
diff --git a/src/parcsr_ls/ams.c b/src/parcsr_ls/ams.c
index 523ef17..95a9229 100644
--- a/src/parcsr_ls/ams.c
+++ b/src/parcsr_ls/ams.c
@@ -324,7 +324,7 @@ HYPRE_Int hypre_ParCSRRelax(/* matrix to relax with */
 
          }
          else
-            hypre_BoomerAMGRelax(A, f, NULL, abs(relax_type), 0, relax_weight,
+            hypre_BoomerAMGRelax(A, f, NULL, hypre_abs(relax_type), 0, relax_weight,
                                  omega, l1_norms, u, v, z);
       }
    }
diff --git a/src/parcsr_ls/par_add_cycle.c b/src/parcsr_ls/par_add_cycle.c
index a631cc6..5fb0e71 100644
--- a/src/parcsr_ls/par_add_cycle.c
+++ b/src/parcsr_ls/par_add_cycle.c
@@ -149,7 +149,7 @@ hypre_BoomerAMGAdditiveCycle( void              *amg_vdata)
             hypre_BoomerAMGRelaxIF(A_array[fine_grid],F_array[fine_grid],
 	     CF_marker_array[fine_grid], rlx_down,rlx_order,1,
              relax_weight[fine_grid], omega[fine_grid],
-             l1_norms_lvl, U_array[fine_grid], Vtemp, Ztemp);
+             l1_norms[level], U_array[fine_grid], Vtemp, Ztemp);
             hypre_ParVectorCopy(F_array[fine_grid],Vtemp);
          }
          else
@@ -351,8 +351,10 @@ HYPRE_Int hypre_CreateLambda(void *amg_vdata)
    HYPRE_Int       num_nonzeros_offd;
 
    HYPRE_Real  **l1_norms_ptr = NULL;
-   HYPRE_Real   *relax_weight = NULL;
-   HYPRE_Real    relax_type;
+   /*HYPRE_Real   *relax_weight = NULL;
+   HYPRE_Int      relax_type; */
+   HYPRE_Int       add_rlx;
+   HYPRE_Real  add_rlx_wt;
 
    /* Acquire data and allocate storage */
 
@@ -362,9 +364,11 @@ HYPRE_Int hypre_CreateLambda(void *amg_vdata)
    additive          = hypre_ParAMGDataAdditive(amg_data);
    mult_additive     = hypre_ParAMGDataMultAdditive(amg_data);
    num_levels        = hypre_ParAMGDataNumLevels(amg_data);
-   relax_weight      = hypre_ParAMGDataRelaxWeight(amg_data);
-   relax_type        = hypre_ParAMGDataGridRelaxType(amg_data)[1];
+   /*relax_weight      = hypre_ParAMGDataRelaxWeight(amg_data);
+   relax_type        = hypre_ParAMGDataGridRelaxType(amg_data)[1];*/
    comm              = hypre_ParCSRMatrixComm(A_array[0]);
+   add_rlx           = hypre_ParAMGDataAddRelaxType(amg_data);
+   add_rlx_wt        = hypre_ParAMGDataAddRelaxWt(amg_data);
 
    hypre_MPI_Comm_size(comm,&num_procs);
 
@@ -705,15 +709,15 @@ HYPRE_Int hypre_CreateLambda(void *amg_vdata)
       }
    
       /* Compute Lambda */ 
-      if (relax_type == 0)
+      if (add_rlx == 0)
       {
-        HYPRE_Real rlx_wt = relax_weight[level];
+        /*HYPRE_Real rlx_wt = relax_weight[level];*/
 #ifdef HYPRE_USING_OPENMP
 #pragma omp for private(i) HYPRE_SMP_SCHEDULE
 #endif
          for (i=0; i < num_rows_tmp; i++)
         {
-           D_data[i] = rlx_wt/A_tmp_diag_data[A_tmp_diag_i[i]];
+           D_data[i] = add_rlx_wt/A_tmp_diag_data[A_tmp_diag_i[i]];
            L_diag_i[cnt_row+i] = start_diag + A_tmp_diag_i[i+1];
            L_offd_i[cnt_row+i] = start_offd + A_tmp_offd_i[i+1];
         }
@@ -838,14 +842,16 @@ HYPRE_Int hypre_CreateDinv(void *amg_vdata)
    HYPRE_Real    *r_data;
    HYPRE_Real    *tmp_data;
    HYPRE_Real    *D_inv = NULL;
-   HYPRE_Real    *relax_weight = NULL;
-   HYPRE_Real     relax_type;
+   /*HYPRE_Real    *relax_weight = NULL;
+   HYPRE_Real     relax_type;*/
 
    HYPRE_Int       addlvl;
    HYPRE_Int       num_levels;
    HYPRE_Int       num_rows_L;
    HYPRE_Int       num_rows_tmp;
    HYPRE_Int       level, i;
+   HYPRE_Int       add_rlx;
+   HYPRE_Real      add_rlx_wt;
 
  /* Local variables  */ 
    HYPRE_Int       Solve_err_flag = 0;
@@ -861,8 +867,10 @@ HYPRE_Int hypre_CreateDinv(void *amg_vdata)
    U_array           = hypre_ParAMGDataUArray(amg_data);
    addlvl            = hypre_ParAMGDataSimple(amg_data);
    num_levels        = hypre_ParAMGDataNumLevels(amg_data);
-   relax_weight      = hypre_ParAMGDataRelaxWeight(amg_data);
-   relax_type        = hypre_ParAMGDataGridRelaxType(amg_data)[1];
+   add_rlx_wt        = hypre_ParAMGDataAddRelaxWt(amg_data);
+   add_rlx           = hypre_ParAMGDataAddRelaxType(amg_data);
+   /*relax_weight      = hypre_ParAMGDataRelaxWeight(amg_data);
+   relax_type        = hypre_ParAMGDataGridRelaxType(amg_data)[1];*/
 
    l1_norms_ptr      = hypre_ParAMGDataL1Norms(amg_data); 
    /* smooth_option       = hypre_ParAMGDataSmoothOption(amg_data); */
@@ -911,16 +919,16 @@ HYPRE_Int hypre_CreateDinv(void *amg_vdata)
       A_tmp_diag = hypre_ParCSRMatrixDiag(A_tmp);
       num_rows_tmp = hypre_CSRMatrixNumRows(A_tmp_diag);
 
-      if (relax_type == 0)
+      if (add_rlx == 0)
       {
-         HYPRE_Real rlx_wt = relax_weight[level];
+         /*HYPRE_Real rlx_wt = relax_weight[level];*/
          HYPRE_Int *A_tmp_diag_i = hypre_CSRMatrixI(A_tmp_diag);
          HYPRE_Real *A_tmp_diag_data = hypre_CSRMatrixData(A_tmp_diag);
 #ifdef HYPRE_USING_OPENMP
 #pragma omp for private(i) HYPRE_SMP_SCHEDULE
 #endif
          for (i=0; i < num_rows_tmp; i++)
-            D_inv[l1_start+i] = rlx_wt/A_tmp_diag_data[A_tmp_diag_i[i]];
+            D_inv[l1_start+i] = add_rlx_wt/A_tmp_diag_data[A_tmp_diag_i[i]];
       }
       else
       {
diff --git a/src/parcsr_ls/par_amg.c b/src/parcsr_ls/par_amg.c
index 0c80501..cb71ec8 100644
--- a/src/parcsr_ls/par_amg.c
+++ b/src/parcsr_ls/par_amg.c
@@ -75,7 +75,9 @@ hypre_BoomerAMGCreate()
    HYPRE_Real   tol;
 
    HYPRE_Int      num_sweeps;  
-   HYPRE_Int      relax_type;   
+   HYPRE_Int      relax_down;   
+   HYPRE_Int      relax_up;   
+   HYPRE_Int      relax_coarse;   
    HYPRE_Int      relax_order;   
    HYPRE_Real   relax_wt;
    HYPRE_Real   outer_wt;
@@ -104,6 +106,8 @@ hypre_BoomerAMGCreate()
    HYPRE_Int        simple;
    HYPRE_Real   add_trunc_factor;
    HYPRE_Int      add_P_max_elmts;
+   HYPRE_Int      add_rlx_type;
+   HYPRE_Real   add_rlx_wt;
 
    /* log info */
    HYPRE_Int      num_iterations;
@@ -137,10 +141,11 @@ hypre_BoomerAMGCreate()
    S_commpkg_switch = 1.0;
    interp_type = 0;
    sep_weight = 0;
-   coarsen_type = 6;
+   coarsen_type = 10;
+   interp_type = 6;
    measure_type = 0;
    setup_type = 1;
-   P_max_elmts = 0;
+   P_max_elmts = 4;
    agg_P_max_elmts = 0;
    agg_P12_max_elmts = 0;
    num_functions = 1;
@@ -185,8 +190,10 @@ hypre_BoomerAMGCreate()
    tol = 1.0e-7;
 
    num_sweeps = 1;
-   relax_type = 3;
-   relax_order = 1;
+   relax_down = 13;
+   relax_up = 14;
+   relax_coarse = 9;
+   relax_order = 0;
    relax_wt = 1.0;
    outer_wt = 1.0;
 
@@ -200,6 +207,8 @@ hypre_BoomerAMGCreate()
    simple = -1;
    add_trunc_factor = 0.0;
    add_P_max_elmts = 0;
+   add_rlx_type = 18;
+   add_rlx_wt = 1.0;
 
    /* log info */
    num_iterations = 0;
@@ -213,6 +222,9 @@ hypre_BoomerAMGCreate()
    debug_flag = 0;
 
    nongalerkin_tol = 0.0;
+
+   HYPRE_ANNOTATION_BEGIN("BoomerAMG.create");
+   
    /*-----------------------------------------------------------------------
     * Create the hypre_ParAMGData structure and return
     *-----------------------------------------------------------------------*/
@@ -234,10 +246,10 @@ hypre_BoomerAMGCreate()
    hypre_BoomerAMGSetAggP12TruncFactor(amg_data, agg_P12_trunc_factor);
    hypre_BoomerAMGSetJacobiTruncThreshold(amg_data, jacobi_trunc_threshold);
    hypre_BoomerAMGSetSCommPkgSwitch(amg_data, S_commpkg_switch);
-   hypre_BoomerAMGSetInterpType(amg_data, interp_type);
    hypre_BoomerAMGSetSepWeight(amg_data, sep_weight);
    hypre_BoomerAMGSetMeasureType(amg_data, measure_type);
    hypre_BoomerAMGSetCoarsenType(amg_data, coarsen_type);
+   hypre_BoomerAMGSetInterpType(amg_data, interp_type);
    hypre_BoomerAMGSetSetupType(amg_data, setup_type);
    hypre_BoomerAMGSetPMaxElmts(amg_data, P_max_elmts);
    hypre_BoomerAMGSetAggPMaxElmts(amg_data, agg_P_max_elmts);
@@ -277,7 +289,9 @@ hypre_BoomerAMGCreate()
    hypre_BoomerAMGSetCycleType(amg_data, cycle_type);
    hypre_BoomerAMGSetTol(amg_data, tol); 
    hypre_BoomerAMGSetNumSweeps(amg_data, num_sweeps);
-   hypre_BoomerAMGSetRelaxType(amg_data, relax_type);
+   hypre_BoomerAMGSetCycleRelaxType(amg_data, relax_down, 1);
+   hypre_BoomerAMGSetCycleRelaxType(amg_data, relax_up, 2);
+   hypre_BoomerAMGSetCycleRelaxType(amg_data, relax_coarse, 3);
    hypre_BoomerAMGSetRelaxOrder(amg_data, relax_order);
    hypre_BoomerAMGSetRelaxWt(amg_data, relax_wt);
    hypre_BoomerAMGSetOuterWt(amg_data, outer_wt);
@@ -295,6 +309,8 @@ hypre_BoomerAMGCreate()
    hypre_BoomerAMGSetSimple(amg_data, simple);
    hypre_BoomerAMGSetMultAddPMaxElmts(amg_data, add_P_max_elmts);
    hypre_BoomerAMGSetMultAddTruncFactor(amg_data, add_trunc_factor);
+   hypre_BoomerAMGSetAddRelaxType(amg_data, add_rlx_type);
+   hypre_BoomerAMGSetAddRelaxWt(amg_data, add_rlx_wt);
    hypre_ParAMGDataLambda(amg_data) = NULL;
    hypre_ParAMGDataXtilde(amg_data) = NULL;
    hypre_ParAMGDataRtilde(amg_data) = NULL;
@@ -381,6 +397,8 @@ hypre_BoomerAMGCreate()
    hypre_ParAMGDataRAP2(amg_data) = 0;
    hypre_ParAMGDataKeepTranspose(amg_data) = 0;
 
+   HYPRE_ANNOTATION_END("BoomerAMG.create");
+   
    return (void *) amg_data;
 }
 
@@ -398,7 +416,10 @@ hypre_BoomerAMGDestroy( void *data )
    void *amg = hypre_ParAMGDataCoarseSolver(amg_data);
    MPI_Comm new_comm = hypre_ParAMGDataNewComm(amg_data);
    HYPRE_Int i;
+   HYPRE_Int *grid_relax_type = hypre_ParAMGDataGridRelaxType(amg_data);
 
+   HYPRE_ANNOTATION_BEGIN("BoomerAMG.destroy");
+   
    if (hypre_ParAMGDataMaxEigEst(amg_data))
    {
       hypre_TFree(hypre_ParAMGDataMaxEigEst(amg_data));
@@ -414,8 +435,19 @@ hypre_BoomerAMGDestroy( void *data )
       hypre_TFree (hypre_ParAMGDataNumGridSweeps(amg_data));
       hypre_ParAMGDataNumGridSweeps(amg_data) = NULL; 
    }
-   if (hypre_ParAMGDataGridRelaxType(amg_data))
+   if (grid_relax_type)
    {
+      HYPRE_Int num_levels = hypre_ParAMGDataNumLevels(amg_data);
+      if (grid_relax_type[1] == 15 || grid_relax_type[3] == 15 )
+      {
+         if (grid_relax_type[1] == 15)
+	    for (i=0; i < num_levels; i++)
+	       HYPRE_ParCSRPCGDestroy(smoother[i]);
+         if (grid_relax_type[3] == 15 && grid_relax_type[1] != 15)
+	    HYPRE_ParCSRPCGDestroy(smoother[num_levels-1]);
+         hypre_TFree(smoother);
+      }
+
       hypre_TFree (hypre_ParAMGDataGridRelaxType(amg_data));
       hypre_ParAMGDataGridRelaxType(amg_data) = NULL; 
    }
@@ -623,6 +655,9 @@ hypre_BoomerAMGDestroy( void *data )
        hypre_MPI_Comm_free (&new_comm);
    }
    hypre_TFree(amg_data);
+
+   HYPRE_ANNOTATION_END("BoomerAMG.destroy");
+   
    return hypre_error_flag;
 }
 
@@ -2683,6 +2718,46 @@ hypre_BoomerAMGSetMultAddPMaxElmts( void     *data,
 }
 
 /*--------------------------------------------------------------------------
+ * Indicates Relaxtion Type for Additive Cycle
+ *--------------------------------------------------------------------------*/
+
+HYPRE_Int
+hypre_BoomerAMGSetAddRelaxType( void     *data,
+                            HYPRE_Int       add_rlx_type )
+{
+   hypre_ParAMGData  *amg_data = (hypre_ParAMGData*) data;
+ 
+   if (!amg_data)
+   {
+      hypre_error_in_arg(1);
+      return hypre_error_flag;
+   } 
+   hypre_ParAMGDataAddRelaxType(amg_data) = add_rlx_type;
+
+   return hypre_error_flag;
+}
+
+/*--------------------------------------------------------------------------
+ * Indicates Relaxation Weight for Additive Cycle
+ *--------------------------------------------------------------------------*/
+
+HYPRE_Int
+hypre_BoomerAMGSetAddRelaxWt( void     *data,
+                            HYPRE_Real       add_rlx_wt )
+{
+   hypre_ParAMGData  *amg_data = (hypre_ParAMGData*) data;
+ 
+   if (!amg_data)
+   {
+      hypre_error_in_arg(1);
+      return hypre_error_flag;
+   } 
+   hypre_ParAMGDataAddRelaxWt(amg_data) = add_rlx_wt;
+
+   return hypre_error_flag;
+}
+
+/*--------------------------------------------------------------------------
  * Indicates max number of elements per row for 1st stage of aggressive
  * coarsening two-stage interpolation
  *--------------------------------------------------------------------------*/
diff --git a/src/parcsr_ls/par_amg.h b/src/parcsr_ls/par_amg.h
index 114955c..0c7cb23 100644
--- a/src/parcsr_ls/par_amg.h
+++ b/src/parcsr_ls/par_amg.h
@@ -210,7 +210,9 @@ typedef struct
    HYPRE_Int      mult_additive;
    HYPRE_Int      simple;
    HYPRE_Int      add_P_max_elmts;
-   HYPRE_Int      add_trunc_factor;
+   HYPRE_Real     add_trunc_factor;
+   HYPRE_Int      add_rlx_type;
+   HYPRE_Real     add_rlx_wt;
    hypre_ParCSRMatrix *Lambda;
    hypre_ParVector *Rtilde;
    hypre_ParVector *Xtilde;
@@ -408,6 +410,8 @@ typedef struct
 #define hypre_ParAMGDataSimple(amg_data) ((amg_data)->simple)
 #define hypre_ParAMGDataMultAddPMaxElmts(amg_data) ((amg_data)->add_P_max_elmts)
 #define hypre_ParAMGDataMultAddTruncFactor(amg_data) ((amg_data)->add_trunc_factor)
+#define hypre_ParAMGDataAddRelaxType(amg_data) ((amg_data)->add_rlx_type)
+#define hypre_ParAMGDataAddRelaxWt(amg_data) ((amg_data)->add_rlx_wt)
 #define hypre_ParAMGDataLambda(amg_data) ((amg_data)->Lambda)
 #define hypre_ParAMGDataRtilde(amg_data) ((amg_data)->Rtilde)
 #define hypre_ParAMGDataXtilde(amg_data) ((amg_data)->Xtilde)
diff --git a/src/parcsr_ls/par_amg_setup.c b/src/parcsr_ls/par_amg_setup.c
index 6bec68b..15b71b8 100644
--- a/src/parcsr_ls/par_amg_setup.c
+++ b/src/parcsr_ls/par_amg_setup.c
@@ -81,6 +81,8 @@ hypre_BoomerAMGSetup( void               *amg_vdata,
    HYPRE_Int      simple = hypre_ParAMGDataSimple(amg_data);
    HYPRE_Int      add_P_max_elmts = hypre_ParAMGDataMultAddPMaxElmts(amg_data);
    HYPRE_Real     add_trunc_factor = hypre_ParAMGDataMultAddTruncFactor(amg_data);
+   HYPRE_Int      add_rlx = hypre_ParAMGDataAddRelaxType(amg_data);
+   HYPRE_Real      add_rlx_wt = hypre_ParAMGDataAddRelaxWt(amg_data);
 
    hypre_ParCSRBlockMatrix **A_block_array, **P_block_array;
  
@@ -244,6 +246,8 @@ hypre_BoomerAMGSetup( void               *amg_vdata,
 
    grid_relax_type[3] = hypre_ParAMGDataUserCoarseRelaxType(amg_data); 
 
+   HYPRE_ANNOTATION_BEGIN("BoomerAMG.setup");
+   
    /* change in definition of standard and multipass interpolation, by
       eliminating interp_type 9 and 5 and setting sep_weight instead
       when using separation of weights option */
@@ -857,12 +861,12 @@ hypre_BoomerAMGSetup( void               *amg_vdata,
 
                if (block_mode)
                {
-                  hypre_BoomerAMGBlockCreateNodalA( A_block_array[level], abs(nodal), nodal_diag, &AN);
+                  hypre_BoomerAMGBlockCreateNodalA( A_block_array[level], hypre_abs(nodal), nodal_diag, &AN);
                }
                else
                {
                   hypre_BoomerAMGCreateNodalA(A_array[level],num_functions,
-                                              dof_func_array[level], abs(nodal), nodal_diag, &AN);
+                                              dof_func_array[level], hypre_abs(nodal), nodal_diag, &AN);
                }
 
                /* dof array not needed for creating S because we pass in that 
@@ -1896,13 +1900,14 @@ hypre_BoomerAMGSetup( void               *amg_vdata,
             HYPRE_Real *d_diag;
             hypre_ParCSRMatrix *Q = NULL;
             Q = hypre_ParMatmul(A_array[level],P);
-            if (grid_relax_type[1] == 0)
+            if (add_rlx == 0)
             {
                hypre_CSRMatrix *lvl_Adiag = hypre_ParCSRMatrixDiag(A_array[level]);
                HYPRE_Int lvl_nrows = hypre_CSRMatrixNumRows(lvl_Adiag);
                HYPRE_Int *lvl_i = hypre_CSRMatrixI(lvl_Adiag);
                HYPRE_Real *lvl_data = hypre_CSRMatrixData(lvl_Adiag);
-               HYPRE_Real w_inv = 1.0/hypre_ParAMGDataRelaxWeight(amg_data)[level];
+               HYPRE_Real w_inv = 1.0/add_rlx_wt;
+               /*HYPRE_Real w_inv = 1.0/hypre_ParAMGDataRelaxWeight(amg_data)[level];*/
                d_diag = hypre_CTAlloc(HYPRE_Real, lvl_nrows);
                for (i=0; i < lvl_nrows; i++)
 		  d_diag[i] = lvl_data[lvl_i[i]]*w_inv;
@@ -2188,7 +2193,8 @@ hypre_BoomerAMGSetup( void               *amg_vdata,
       hypre_ParAMGDataSmoother(amg_data) = smoother;
    }
 
-   for (j = 0; j < num_levels; j++)
+   if (addlvl == -1) addlvl = num_levels;
+   for (j = 0; j < addlvl; j++)
    {
       if (num_threads == 1)
       {
@@ -2217,8 +2223,6 @@ hypre_BoomerAMGSetup( void               *amg_vdata,
          {
             hypre_ParCSRComputeL1Norms(A_array[j], 1, NULL, &l1_norms[j]);
          }
-         else if (addlvl > -1 && level >= addlvl)
-            hypre_ParCSRComputeL1Norms(A_array[j], 1, NULL, &l1_norms[j]);
       }
       else
       {
@@ -2247,10 +2251,20 @@ hypre_BoomerAMGSetup( void               *amg_vdata,
          {
             hypre_ParCSRComputeL1NormsThreads(A_array[j], 1, num_threads, NULL, &l1_norms[j]);
          }
-         else if (addlvl > -1 && level >= addlvl)
-            hypre_ParCSRComputeL1NormsThreads(A_array[j], 1, num_threads, NULL, &l1_norms[j]);
-
       }
+   }
+   for (j = addlvl; j < num_levels; j++)
+   {
+      if (add_rlx == 18 )
+      {
+         if (num_threads == 1)
+               hypre_ParCSRComputeL1Norms(A_array[j], 1, NULL, &l1_norms[j]);
+         else
+               hypre_ParCSRComputeL1NormsThreads(A_array[j], 1, num_threads, NULL, &l1_norms[j]);
+      }
+   }
+   for (j = 0; j < num_levels; j++)
+   {
       if (grid_relax_type[1] == 16 || grid_relax_type[2] == 16 || (grid_relax_type[3] == 16 && j== (num_levels-1)))
       {
          HYPRE_Int scale = 1;
@@ -2263,10 +2277,10 @@ hypre_BoomerAMGSetup( void               *amg_vdata,
      {
         
         HYPRE_ParCSRPCGCreate(comm, &smoother[j]);
-        HYPRE_ParCSRPCGSetup(smoother[j],
+        /*HYPRE_ParCSRPCGSetup(smoother[j],
                              (HYPRE_ParCSRMatrix) A_array[j],
                              (HYPRE_ParVector) F_array[j],
-                             (HYPRE_ParVector) U_array[j]);
+                             (HYPRE_ParVector) U_array[j]);*/
         
         HYPRE_PCGSetTol(smoother[j], 1e-12); /* make small */
         HYPRE_PCGSetTwoNorm(smoother[j], 1); /* use 2-norm*/
@@ -2564,5 +2578,7 @@ hypre_BoomerAMGSetup( void               *amg_vdata,
 }
 #endif
 
+   HYPRE_ANNOTATION_END("BoomerAMG.setup");
+ 
    return(hypre_error_flag);
 }  
diff --git a/src/parcsr_ls/par_amg_solve.c b/src/parcsr_ls/par_amg_solve.c
index 9e5fd57..8c0e210 100644
--- a/src/parcsr_ls/par_amg_solve.c
+++ b/src/parcsr_ls/par_amg_solve.c
@@ -89,6 +89,8 @@ hypre_BoomerAMGSolve( void               *amg_vdata,
    hypre_ParVector  *Vtemp;
    hypre_ParVector  *Residual;
 
+   HYPRE_ANNOTATION_BEGIN("BoomerAMG.solve");
+      
    hypre_MPI_Comm_size(comm, &num_procs);   
    hypre_MPI_Comm_rank(comm,&my_id);
 
@@ -127,7 +129,6 @@ hypre_BoomerAMGSolve( void               *amg_vdata,
 */
    Vtemp = hypre_ParAMGDataVtemp(amg_data);
 
-
    /*-----------------------------------------------------------------------
     *    Write the solver parameters
     *-----------------------------------------------------------------------*/
@@ -160,7 +161,7 @@ hypre_BoomerAMGSolve( void               *amg_vdata,
     *    Compute initial fine-grid residual and print 
     *-----------------------------------------------------------------------*/
 
-   if (amg_print_level > 1 || amg_logging > 1)
+   if (amg_print_level > 1 || amg_logging > 1 || tol > 0.)
    {
      if ( amg_logging > 1 ) {
         hypre_ParVectorCopy(F_array[0], Residual );
@@ -193,6 +194,7 @@ hypre_BoomerAMGSolve( void               *amg_vdata,
           hypre_printf("ERROR detected by Hypre ...  END\n\n\n");
         }
         hypre_error(HYPRE_ERROR_GENERIC);
+        HYPRE_ANNOTATION_END("BoomerAMG.solve");
         return hypre_error_flag;
      }
 
@@ -361,6 +363,8 @@ hypre_BoomerAMGSolve( void               *amg_vdata,
       hypre_TFree(num_variables);
    }
 
+   HYPRE_ANNOTATION_END("BoomerAMG.solve");
+   
    return hypre_error_flag;
 }
 
diff --git a/src/parcsr_ls/par_amg_solveT.c b/src/parcsr_ls/par_amg_solveT.c
index 44b7408..85f9403 100644
--- a/src/parcsr_ls/par_amg_solveT.c
+++ b/src/parcsr_ls/par_amg_solveT.c
@@ -106,6 +106,8 @@ hypre_BoomerAMGSolveT( void               *amg_vdata,
    F_array[0] = f;
    U_array[0] = u;
 
+   HYPRE_ANNOTATION_BEGIN("BoomerAMG.solveT");
+
 /*   Vtemp = hypre_ParVectorCreate(hypre_ParCSRMatrixComm(A_array[0]),
                                  hypre_ParCSRMatrixGlobalNumRows(A_array[0]),
                                  hypre_ParCSRMatrixRowStarts(A_array[0]));
@@ -290,6 +292,8 @@ hypre_BoomerAMGSolveT( void               *amg_vdata,
    hypre_TFree(num_coeffs);
    hypre_TFree(num_variables);
 
+   HYPRE_ANNOTATION_END("BoomerAMG.solveT");
+
    return(Solve_err_flag);
 }
 
@@ -357,7 +361,9 @@ hypre_BoomerAMGCycleT( void              *amg_vdata,
    HYPRE_Real   *D_mat;
    HYPRE_Real   *S_vec;
 #endif
-   
+
+   HYPRE_ANNOTATION_BEGIN("BoomerAMG.cycleT");
+
    /* Acquire data and allocate storage */
 
    A_array           = hypre_ParAMGDataAArray(amg_data);
@@ -488,6 +494,7 @@ hypre_BoomerAMGCycleT( void              *amg_vdata,
          {
             hypre_TFree(lev_counter);
             hypre_TFree(num_coeffs);
+            HYPRE_ANNOTATION_END("BoomerAMG.cycleT");
             return(Solve_err_flag);
          }
       }
@@ -565,6 +572,8 @@ hypre_BoomerAMGCycleT( void              *amg_vdata,
    hypre_TFree(lev_counter);
    hypre_TFree(num_coeffs);
 
+   HYPRE_ANNOTATION_END("BoomerAMG.cycleT");
+
    return(Solve_err_flag);
 }
 
@@ -618,7 +627,9 @@ HYPRE_Int  hypre_BoomerAMGRelaxT( hypre_ParCSRMatrix *A,
    HYPRE_Real     *b_vec;
 
    HYPRE_Real      zero = 0.0;
-  
+
+   HYPRE_ANNOTATION_BEGIN("BoomerAMG.relaxT");
+      
    /*-----------------------------------------------------------------------
     * Switch statement to direct control based on relax_type:
     *     relax_type = 7 -> Jacobi (uses ParMatvec)
@@ -711,5 +722,7 @@ HYPRE_Int  hypre_BoomerAMGRelaxT( hypre_ParCSRMatrix *A,
       break;   
    }
 
+   HYPRE_ANNOTATION_END("BoomerAMG.relaxT");
+
    return(relax_error); 
 }
diff --git a/src/parcsr_ls/par_coarsen.c b/src/parcsr_ls/par_coarsen.c
index db91bfb..96fda4b 100644
--- a/src/parcsr_ls/par_coarsen.c
+++ b/src/parcsr_ls/par_coarsen.c
@@ -2524,7 +2524,7 @@ hypre_BoomerAMGCoarsenPMIS( hypre_ParCSRMatrix    *S,
         {
            i = graph_array[ig];
 
-           if (!CF_marker[i]==0) /* C or F point */
+           if (CF_marker[i]!=0) /* C or F point */
            {
               /* the independent set subroutine needs measure 0 for
                  removed nodes */
@@ -2540,7 +2540,7 @@ hypre_BoomerAMGCoarsenPMIS( hypre_ParCSRMatrix    *S,
         {
            i = graph_array_offd[ig];
 
-           if (!CF_marker_offd[i]==0) /* C of F point */
+           if (CF_marker_offd[i]!=0) /* C of F point */
            {
               /* the independent set subroutine needs measure 0 for
                  removed nodes */
diff --git a/src/parcsr_ls/par_cycle.c b/src/parcsr_ls/par_cycle.c
index 686e9d1..1f09407 100644
--- a/src/parcsr_ls/par_cycle.c
+++ b/src/parcsr_ls/par_cycle.c
@@ -24,12 +24,16 @@
 #include "par_amg.h"
 #include "par_csr_block_matrix.h"
 
+#ifdef HYPRE_USING_CALIPER
+#include <caliper/cali.h>
+#endif
+
 /*--------------------------------------------------------------------------
  * hypre_BoomerAMGCycle
  *--------------------------------------------------------------------------*/
 
 HYPRE_Int
-hypre_BoomerAMGCycle( void              *amg_vdata, 
+hypre_BoomerAMGCycle( void              *amg_vdata,
                    hypre_ParVector  **F_array,
                    hypre_ParVector  **U_array   )
 {
@@ -60,24 +64,24 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
    HYPRE_Int     **point_map_array;
    HYPRE_Int     **v_at_point_array; */
 
-   HYPRE_Real    cycle_op_count;   
+   HYPRE_Real    cycle_op_count;
    HYPRE_Int       cycle_type;
    HYPRE_Int       num_levels;
    HYPRE_Int       max_levels;
 
    HYPRE_Real   *num_coeffs;
-   HYPRE_Int      *num_grid_sweeps;   
-   HYPRE_Int      *grid_relax_type;   
-   HYPRE_Int     **grid_relax_points;  
+   HYPRE_Int      *num_grid_sweeps;
+   HYPRE_Int      *grid_relax_type;
+   HYPRE_Int     **grid_relax_points;
 
    HYPRE_Int     block_mode;
-   
+
    HYPRE_Real  *max_eig_est;
    HYPRE_Real  *min_eig_est;
    HYPRE_Int      cheby_order;
    HYPRE_Real   cheby_fraction;
 
- /* Local variables  */ 
+ /* Local variables  */
    HYPRE_Int      *lev_counter;
    HYPRE_Int       Solve_err_flag;
    HYPRE_Int       k;
@@ -116,6 +120,11 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
    HYPRE_Real   *D_mat;
    HYPRE_Real   *S_vec;
 #endif
+
+#ifdef HYPRE_USING_CALIPER
+   cali_id_t iter_attr =
+     cali_create_attribute("hypre.par_cycle.level", CALI_TYPE_INT, CALI_ATTR_DEFAULT);
+#endif
    
    /* Acquire data and allocate storage */
 
@@ -142,11 +151,11 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
    grid_relax_type     = hypre_ParAMGDataGridRelaxType(amg_data);
    grid_relax_points   = hypre_ParAMGDataGridRelaxPoints(amg_data);
    relax_order         = hypre_ParAMGDataRelaxOrder(amg_data);
-   relax_weight        = hypre_ParAMGDataRelaxWeight(amg_data); 
-   omega               = hypre_ParAMGDataOmega(amg_data); 
-   smooth_type         = hypre_ParAMGDataSmoothType(amg_data); 
-   smooth_num_levels   = hypre_ParAMGDataSmoothNumLevels(amg_data); 
-   l1_norms            = hypre_ParAMGDataL1Norms(amg_data); 
+   relax_weight        = hypre_ParAMGDataRelaxWeight(amg_data);
+   omega               = hypre_ParAMGDataOmega(amg_data);
+   smooth_type         = hypre_ParAMGDataSmoothType(amg_data);
+   smooth_num_levels   = hypre_ParAMGDataSmoothNumLevels(amg_data);
+   l1_norms            = hypre_ParAMGDataL1Norms(amg_data);
    /* smooth_option       = hypre_ParAMGDataSmoothOption(amg_data); */
 
    max_eig_est = hypre_ParAMGDataMaxEigEst(amg_data);
@@ -175,28 +184,28 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
    {
       for (j = 1; j < num_levels; j++)
          num_coeffs[j] = hypre_ParCSRBlockMatrixNumNonzeros(A_block_array[j]);
-      
+
    }
-   else 
+   else
    {
        for (j = 1; j < num_levels; j++)
          num_coeffs[j] = hypre_ParCSRMatrixDNumNonzeros(A_array[j]);
    }
-   
+
    /*---------------------------------------------------------------------
     *    Initialize cycling control counter
     *
     *     Cycling is controlled using a level counter: lev_counter[k]
-    *     
+    *
     *     Each time relaxation is performed on level k, the
     *     counter is decremented by 1. If the counter is then
     *     negative, we go to the next finer level. If non-
     *     negative, we go to the next coarser level. The
     *     following actions control cycling:
-    *     
+    *
     *     a. lev_counter[0] is initialized to 1.
     *     b. lev_counter[k] is initialized to cycle_type for k>0.
-    *     
+    *
     *     c. During cycling, when going down to level k, lev_counter[k]
     *        is set to the max of (lev_counter[k],cycle_type)
     *---------------------------------------------------------------------*/
@@ -204,7 +213,7 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
    Not_Finished = 1;
 
    lev_counter[0] = 1;
-   for (k = 1; k < num_levels; ++k) 
+   for (k = 1; k < num_levels; ++k)
    {
       lev_counter[k] = cycle_type;
    }
@@ -224,7 +233,7 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
          Utemp = hypre_ParVectorCreate(comm,hypre_ParVectorGlobalSize(Vtemp),
                         hypre_ParVectorPartitioning(Vtemp));
          hypre_ParVectorOwnsPartitioning(Utemp) = 0;
-         local_size 
+         local_size
             = hypre_VectorSize(hypre_ParVectorLocalVector(Vtemp));
          if (local_size < actual_local_size)
          {
@@ -236,17 +245,21 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
 	     hypre_ParVectorInitialize(Utemp);
       }
    }
-   
-  
+
+
    /*---------------------------------------------------------------------
     * Main loop of cycling
     *--------------------------------------------------------------------*/
-  
+
+#ifdef HYPRE_USING_CALIPER
+   cali_set_int(iter_attr, level);
+#endif
+   
    while (Not_Finished)
    {
-      if (num_levels > 1) 
+      if (num_levels > 1)
       {
-        local_size 
+        local_size
             = hypre_VectorSize(hypre_ParVectorLocalVector(F_array[level]));
         hypre_VectorSize(hypre_ParVectorLocalVector(Vtemp)) = local_size;
         if (smooth_num_levels <= level)
@@ -266,14 +279,14 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
            hypre_ParVectorSetConstantValues(Ztemp,0);
            alpha = -1.0;
            beta = 1.0;
-           hypre_ParCSRMatrixMatvecOutOfPlace(alpha, A_array[level], 
+           hypre_ParCSRMatrixMatvecOutOfPlace(alpha, A_array[level],
                                 U_array[level], beta, F_array[level], Rtemp);
 	   cg_num_sweep = hypre_ParAMGDataSmoothNumSweeps(amg_data);
            num_sweep = num_grid_sweeps[cycle_param];
            Aux_U = Ztemp;
            Aux_F = Rtemp;
 	}
-	else 
+	else
 	{
            cg_num_sweep = 1;
 	   num_sweep = hypre_ParAMGDataSmoothNumSweeps(amg_data);
@@ -291,6 +304,7 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
         /* TK: Use the user relax type (instead of 0) to allow for setting a
            convergent smoother (e.g. in the solution of singular problems). */
         relax_type = hypre_ParAMGDataUserRelaxType(amg_data);
+        if (relax_type == -1) relax_type = 6;
       }
 
       if (l1_norms != NULL)
@@ -304,7 +318,7 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
       }
       else
       {
-         
+
         /*------------------------------------------------------------------
          * Do the relaxation num_sweep times
          *-----------------------------------------------------------------*/
@@ -337,21 +351,21 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
                     case 1:
                     cycle_op_count += num_coeffs[level+1];
                     break;
-  
-                    case -1: 
-                    cycle_op_count += (num_coeffs[level]-num_coeffs[level+1]); 
+
+                    case -1:
+                    cycle_op_count += (num_coeffs[level]-num_coeffs[level+1]);
                     break;
                  }
               }
 	      else
               {
-                 cycle_op_count += num_coeffs[level]; 
+                 cycle_op_count += num_coeffs[level];
               }
               /*-----------------------------------------------
                 Choose Smoother
                 -----------------------------------------------*/
 
-              if (smooth_num_levels > level && 
+              if (smooth_num_levels > level &&
 			(smooth_type == 7 || smooth_type == 8 ||
 			smooth_type == 9 || smooth_type == 19 ||
 			smooth_type == 17 || smooth_type == 18))
@@ -359,7 +373,7 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
                  hypre_VectorSize(hypre_ParVectorLocalVector(Utemp)) = local_size;
                  alpha = -1.0;
                  beta = 1.0;
-                 hypre_ParCSRMatrixMatvecOutOfPlace(alpha, A_array[level], 
+                 hypre_ParCSRMatrixMatvecOutOfPlace(alpha, A_array[level],
                                 U_array[level], beta, Aux_F, Vtemp);
                  if (smooth_type == 8 || smooth_type == 18)
                     HYPRE_ParCSRParaSailsSolve(smoother[level],
@@ -421,7 +435,7 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
                  else /* not CF - so use through AMS */
                  {
                     if (num_threads == 1)
-                       hypre_ParCSRRelax(A_array[level], 
+                       hypre_ParCSRRelax(A_array[level],
                                        Aux_F,
                                        1,
                                        1,
@@ -429,11 +443,11 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
                                        relax_weight[level],
                                        omega[level],0,0,0,0,
                                        Aux_U,
-                                       Vtemp, 
+                                       Vtemp,
                                        Ztemp);
 
                     else
-                       hypre_ParCSRRelaxThreads(A_array[level], 
+                       hypre_ParCSRRelaxThreads(A_array[level],
                                               Aux_F,
                                               1,
                                               1,
@@ -449,8 +463,8 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
               {  /* CG */
                  if (j ==0) /* do num sweep iterations of CG */
                     hypre_ParCSRRelax_CG( smoother[level],
-                                        A_array[level], 
-                                        Aux_F,      
+                                        A_array[level],
+                                        Aux_F,
                                         Aux_U,
                                         num_sweep);
               }
@@ -458,16 +472,16 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
               { /* scaled Chebyshev */
                  HYPRE_Int scale = 1;
                  HYPRE_Int variant = 0;
-                 hypre_ParCSRRelax_Cheby(A_array[level], 
+                 hypre_ParCSRRelax_Cheby(A_array[level],
                                        Aux_F,
-                                       max_eig_est[level],     
-                                       min_eig_est[level],     
+                                       max_eig_est[level],
+                                       min_eig_est[level],
                                        cheby_fraction, cheby_order, scale,
                                        variant, Aux_U, Vtemp, Ztemp );
               }
               else if (relax_type ==17)
               {
-                 hypre_BoomerAMGRelax_FCFJacobi(A_array[level], 
+                 hypre_BoomerAMGRelax_FCFJacobi(A_array[level],
                                               Aux_F,
                                               CF_marker_array[level],
                                               relax_weight[level],
@@ -476,7 +490,7 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
               }
 	      else if (old_version)
 	      {
-                 Solve_err_flag = hypre_BoomerAMGRelax(A_array[level], 
+                 Solve_err_flag = hypre_BoomerAMGRelax(A_array[level],
                                                      Aux_F,
                                                      CF_marker_array[level],
                                                      relax_type, relax_points,
@@ -484,15 +498,15 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
                                                      omega[level],
                                                      l1_norms_level,
                                                      Aux_U,
-                                                     Vtemp, 
+                                                     Vtemp,
                                                      Ztemp);
 	      }
-	      else 
+	      else
 	      {
                  /* smoother than can have CF ordering */
                  if (block_mode)
                  {
-                     Solve_err_flag = hypre_BoomerAMGBlockRelaxIF(A_block_array[level], 
+                     Solve_err_flag = hypre_BoomerAMGBlockRelaxIF(A_block_array[level],
                                                                   Aux_F,
                                                                   CF_marker_array[level],
                                                                   relax_type,
@@ -505,7 +519,7 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
                  }
                  else
                  {
-                    Solve_err_flag = hypre_BoomerAMGRelaxIF(A_array[level], 
+                    Solve_err_flag = hypre_BoomerAMGRelaxIF(A_array[level],
                                                           Aux_F,
                                                           CF_marker_array[level],
                                                           relax_type,
@@ -515,11 +529,11 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
                                                           omega[level],
                                                           l1_norms_level,
                                                           Aux_U,
-                                                          Vtemp, 
+                                                          Vtemp,
                                                           Ztemp);
                  }
 	      }
- 
+
               if (Solve_err_flag != 0)
                  return(Solve_err_flag);
            }
@@ -548,12 +562,12 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
        *-----------------------------------------------------------------*/
 
       --lev_counter[level];
-       
+
       if (lev_counter[level] >= 0 && level != num_levels-1)
       {
-                               
+
          /*---------------------------------------------------------------
-          * Visit coarser level next.  
+          * Visit coarser level next.
  	  * Compute residual using hypre_ParCSRMatrixMatvec.
           * Perform restriction using hypre_ParCSRMatrixMatvecT.
           * Reset counters and cycling parameters for coarse level
@@ -562,8 +576,8 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
          fine_grid = level;
          coarse_grid = level + 1;
 
-         hypre_ParVectorSetConstantValues(U_array[coarse_grid], 0.0); 
-          
+         hypre_ParVectorSetConstantValues(U_array[coarse_grid], 0.0);
+
          alpha = -1.0;
          beta = 1.0;
 
@@ -573,7 +587,7 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
             hypre_ParCSRBlockMatrixMatvec(alpha, A_block_array[fine_grid], U_array[fine_grid],
                                           beta, Vtemp);
          }
-         else 
+         else
          {
             // JSP: avoid unnecessary copy using out-of-place version of SpMV
             hypre_ParCSRMatrixMatvecOutOfPlace(alpha, A_array[fine_grid], U_array[fine_grid],
@@ -598,6 +612,10 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
          lev_counter[level] = hypre_max(lev_counter[level],cycle_type);
          cycle_param = 1;
          if (level == num_levels-1) cycle_param = 3;
+
+#ifdef HYPRE_USING_CALIPER
+         cali_set_int(iter_attr, level);  /* set the level for caliper here */
+#endif
       }
 
       else if (level != 0)
@@ -614,19 +632,23 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
          beta = 1.0;
          if (block_mode)
          {
-            hypre_ParCSRBlockMatrixMatvec(alpha, P_block_array[fine_grid], 
+            hypre_ParCSRBlockMatrixMatvec(alpha, P_block_array[fine_grid],
                                      U_array[coarse_grid],
-                                     beta, U_array[fine_grid]);   
+                                     beta, U_array[fine_grid]);
          }
-         else 
+         else
          {
-            hypre_ParCSRMatrixMatvec(alpha, P_array[fine_grid], 
+            hypre_ParCSRMatrixMatvec(alpha, P_array[fine_grid],
                                      U_array[coarse_grid],
-                                     beta, U_array[fine_grid]);            
+                                     beta, U_array[fine_grid]);
          }
-         
+
          --level;
          cycle_param = 2;
+         
+#ifdef HYPRE_USING_CALIPER
+         cali_set_int(iter_attr, level);  /* set the level for caliper here */
+#endif
       }
       else
       {
@@ -634,13 +656,17 @@ hypre_BoomerAMGCycle( void              *amg_vdata,
       }
    }
 
+#ifdef HYPRE_USING_CALIPER
+   cali_end(iter_attr);  /* unset "iter" */
+#endif
+   
    hypre_ParAMGDataCycleOpCount(amg_data) = cycle_op_count;
 
    hypre_TFree(lev_counter);
    hypre_TFree(num_coeffs);
    if (smooth_num_levels > 0)
    {
-     if (smooth_type == 7 || smooth_type == 8 || smooth_type == 9 || 
+     if (smooth_type == 7 || smooth_type == 8 || smooth_type == 9 ||
 	smooth_type == 17 || smooth_type == 18 || smooth_type == 19 )
         hypre_ParVectorDestroy(Utemp);
    }
diff --git a/src/parcsr_ls/par_laplace.c b/src/parcsr_ls/par_laplace.c
index 9f57203..2fd5295 100644
--- a/src/parcsr_ls/par_laplace.c
+++ b/src/parcsr_ls/par_laplace.c
@@ -835,7 +835,7 @@ GenerateSysLaplacian( MPI_Comm comm,
       hypre_qsort0(col_map_offd, 0, num_cols_offd-1);
 
       for (i=0; i < num_fun*num_cols_offd; i++)
-         for (j=hypre_min(0,abs(i-num_fun)); j < num_cols_offd; j++)
+         for (j=hypre_min(0,hypre_abs(i-num_fun)); j < num_cols_offd; j++)
             if (offd_j[i] == col_map_offd[j])
             {
                offd_j[i] = j;
@@ -1343,7 +1343,7 @@ GenerateSysLaplacianVCoef( MPI_Comm comm,
       hypre_qsort0(col_map_offd, 0, num_cols_offd-1);
 
       for (i=0; i < num_fun*num_cols_offd; i++)
-         for (j=hypre_min(0,abs(i-num_fun)); j < num_cols_offd; j++)
+         for (j=hypre_min(0,hypre_abs(i-num_fun)); j < num_cols_offd; j++)
             if (offd_j[i] == col_map_offd[j])
             {
                offd_j[i] = j;
diff --git a/src/parcsr_ls/par_nodal_systems.c b/src/parcsr_ls/par_nodal_systems.c
index 7748e02..065580f 100644
--- a/src/parcsr_ls/par_nodal_systems.c
+++ b/src/parcsr_ls/par_nodal_systems.c
@@ -130,7 +130,7 @@ hypre_BoomerAMGCreateNodalA(hypre_ParCSRMatrix    *A,
       comm_pkg = hypre_ParCSRMatrixCommPkg(A);
    }
 
-   mode = abs(option);
+   mode = hypre_abs(option);
 
    comm_pkg_AN = NULL;
    col_map_offd_AN = NULL;
diff --git a/src/parcsr_ls/par_rap.c b/src/parcsr_ls/par_rap.c
index 30f1777..adbe585 100644
--- a/src/parcsr_ls/par_rap.c
+++ b/src/parcsr_ls/par_rap.c
@@ -612,7 +612,7 @@ hypre_BoomerAMGBuildCoarseOperatorKT( hypre_ParCSRMatrix  *RT,
 #pragma omp parallel for HYPRE_SMP_SCHEDULE
       for (i=0 ; i < P_ext_offd_size; i++)
          P_ext_offd_j[i] = hypre_UnorderedIntMapGet(&col_map_offd_Pext_inverse, P_ext_offd_j[i]);
-      hypre_UnorderedIntMapDestroy(&col_map_offd_Pext_inverse);
+      if (num_cols_offd_Pext) hypre_UnorderedIntMapDestroy(&col_map_offd_Pext_inverse);
    }
 #else /* !HYPRE_CONCURRENT_HOPSCOTCH */
    if (P_ext_offd_size || num_cols_offd_P)
diff --git a/src/parcsr_ls/par_relax.c b/src/parcsr_ls/par_relax.c
index 31118d5..66e6b0c 100644
--- a/src/parcsr_ls/par_relax.c
+++ b/src/parcsr_ls/par_relax.c
@@ -3108,29 +3108,29 @@ HYPRE_Int  hypre_BoomerAMGRelax( hypre_ParCSRMatrix *A,
           *-----------------------------------------------------------------*/
         if (num_procs > 1)
         {
-        num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
+         num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
 
-        v_buf_data = hypre_CTAlloc(HYPRE_Real,
+         v_buf_data = hypre_CTAlloc(HYPRE_Real,
                         hypre_ParCSRCommPkgSendMapStart(comm_pkg, num_sends));
 
-        Vext_data = hypre_CTAlloc(HYPRE_Real,num_cols_offd);
+         Vext_data = hypre_CTAlloc(HYPRE_Real,num_cols_offd);
 
-        if (num_cols_offd)
-        {
+         if (num_cols_offd)
+         {
                 A_offd_j = hypre_CSRMatrixJ(A_offd);
                 A_offd_data = hypre_CSRMatrixData(A_offd);
-        }
+         }
 
-        index = 0;
-        for (i = 0; i < num_sends; i++)
-        {
+         index = 0;
+         for (i = 0; i < num_sends; i++)
+         {
                 start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i);
                 for (j=start; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg,i+1); j++)
                         v_buf_data[index++]
                         = u_data[hypre_ParCSRCommPkgSendMapElmt(comm_pkg,j)];
-        }
+         }
 
-        comm_handle = hypre_ParCSRCommHandleCreate( 1, comm_pkg, v_buf_data,
+         comm_handle = hypre_ParCSRCommHandleCreate( 1, comm_pkg, v_buf_data,
                 Vext_data);
 
          /*-----------------------------------------------------------------
diff --git a/src/parcsr_ls/par_rotate_7pt.c b/src/parcsr_ls/par_rotate_7pt.c
index a31b714..e718d76 100644
--- a/src/parcsr_ls/par_rotate_7pt.c
+++ b/src/parcsr_ls/par_rotate_7pt.c
@@ -368,6 +368,7 @@ GenerateRotate7pt( MPI_Comm comm,
          }
       }
 
+      num_cols_offd = cnt+1;
       for (i=0; i < o_cnt; i++)
       {
          for (j=0; j < num_cols_offd; j++)
diff --git a/src/parcsr_ls/par_schwarz.c b/src/parcsr_ls/par_schwarz.c
index 0cf4280..0c31ae3 100644
--- a/src/parcsr_ls/par_schwarz.c
+++ b/src/parcsr_ls/par_schwarz.c
@@ -136,20 +136,23 @@ hypre_SchwarzSetup(void               *schwarz_vdata,
                                   num_functions, dof_func,
                                   &domain_structure, &pivots, use_nonsymm);
 
-      if (variant == 2)
+      if (domain_structure)
       {
+       if (variant == 2)
+       {
          hypre_ParGenerateScale(A, domain_structure, relax_weight,
 		&scale);
          hypre_SchwarzDataScale(schwarz_data) = scale;
-      }
-      else
-      {
+       }
+       else
+       {
          hypre_ParGenerateHybridScale(A, domain_structure, &A_boundary, &scale);
          hypre_SchwarzDataScale(schwarz_data) = scale;
          if (hypre_CSRMatrixNumCols(hypre_ParCSRMatrixOffd(A)))
             hypre_SchwarzDataABoundary(schwarz_data) = A_boundary;
          else
             hypre_SchwarzDataABoundary(schwarz_data) = NULL;
+       }
       }
    }
    else
@@ -158,12 +161,15 @@ hypre_SchwarzSetup(void               *schwarz_vdata,
                                 domain_type, overlap,
                                 num_functions, dof_func,
                                 &domain_structure, &pivots, use_nonsymm);
-      if (variant == 1)
+      if (domain_structure)
       {
+       if (variant == 1)
+       {
          hypre_GenerateScale(domain_structure, 
 		hypre_CSRMatrixNumRows(hypre_ParCSRMatrixDiag(A)),
 		relax_weight, &scale);
          hypre_SchwarzDataScale(schwarz_data) = scale;
+       }
       }
    }
 
@@ -197,30 +203,33 @@ hypre_SchwarzSolve(void               *schwarz_vdata,
    
    HYPRE_Int *pivots = hypre_SchwarzDataPivots(schwarz_data);
 
-   if (variant == 2)
+   if (domain_structure)
    {
+    if (variant == 2)
+    {
       hypre_ParAdSchwarzSolve(A, f, domain_structure, scale, u, Vtemp, pivots, use_nonsymm);
-   }
-   else if (variant == 3)
-   {
+    }
+    else if (variant == 3)
+    {
       hypre_ParMPSchwarzSolve(A, A_boundary, f, domain_structure, u,
                               relax_wt, scale, Vtemp, pivots, use_nonsymm); 
-   }
-   else if (variant == 1)
-   {
+    }
+    else if (variant == 1)
+    {
       hypre_AdSchwarzSolve(A, f, domain_structure, scale, u, Vtemp, pivots, use_nonsymm);
-   }
-   else if (variant == 4)
-   {
+    }
+    else if (variant == 4)
+    {
       hypre_MPSchwarzFWSolve(A, hypre_ParVectorLocalVector(f), 
 				domain_structure, u, relax_wt, 
                              hypre_ParVectorLocalVector(Vtemp), pivots, use_nonsymm);
-   }
-   else 
-   {
+    }
+    else 
+    {
       hypre_MPSchwarzSolve(A, hypre_ParVectorLocalVector(f), 
                                   domain_structure, u, relax_wt, 
                                   hypre_ParVectorLocalVector(Vtemp), pivots, use_nonsymm);
+    }
    }
       
    return hypre_error_flag;
diff --git a/src/parcsr_ls/par_stats.c b/src/parcsr_ls/par_stats.c
index 4a0b9b2..8bb9b96 100644
--- a/src/parcsr_ls/par_stats.c
+++ b/src/parcsr_ls/par_stats.c
@@ -155,6 +155,8 @@ hypre_BoomerAMGSetupStats( void               *amg_vdata,
    HYPRE_Int additive;
    HYPRE_Int mult_additive;
    HYPRE_Int simple;
+   HYPRE_Int add_rlx;
+   HYPRE_Real add_rlx_wt;
  
    hypre_MPI_Comm_size(comm, &num_procs);   
    hypre_MPI_Comm_rank(comm,&my_id);
@@ -173,6 +175,8 @@ hypre_BoomerAMGSetupStats( void               *amg_vdata,
    additive = hypre_ParAMGDataAdditive(amg_data);
    mult_additive = hypre_ParAMGDataMultAdditive(amg_data);
    simple = hypre_ParAMGDataSimple(amg_data);
+   add_rlx = hypre_ParAMGDataAddRelaxType(amg_data);
+   add_rlx_wt = hypre_ParAMGDataAddRelaxWt(amg_data);
 
 
    A_block_array = hypre_ParAMGDataABlockArray(amg_data);
@@ -220,55 +224,55 @@ hypre_BoomerAMGSetupStats( void               *amg_vdata,
       {
 	hypre_printf(" Coarsening Type = Cleary-Luby-Jones-Plassman\n");
       }
-      else if (abs(coarsen_type) == 1) 
+      else if (hypre_abs(coarsen_type) == 1) 
       {
 	hypre_printf(" Coarsening Type = Ruge\n");
       }
-      else if (abs(coarsen_type) == 2) 
+      else if (hypre_abs(coarsen_type) == 2) 
       {
 	hypre_printf(" Coarsening Type = Ruge2B\n");
       }
-      else if (abs(coarsen_type) == 3) 
+      else if (hypre_abs(coarsen_type) == 3) 
       {
 	hypre_printf(" Coarsening Type = Ruge3\n");
       }
-      else if (abs(coarsen_type) == 4) 
+      else if (hypre_abs(coarsen_type) == 4) 
       {
 	hypre_printf(" Coarsening Type = Ruge 3c \n");
       }
-      else if (abs(coarsen_type) == 5) 
+      else if (hypre_abs(coarsen_type) == 5) 
       {
 	hypre_printf(" Coarsening Type = Ruge relax special points \n");
       }
-      else if (abs(coarsen_type) == 6) 
+      else if (hypre_abs(coarsen_type) == 6) 
       {
 	hypre_printf(" Coarsening Type = Falgout-CLJP \n");
       }
-      else if (abs(coarsen_type) == 8) 
+      else if (hypre_abs(coarsen_type) == 8) 
       {
 	hypre_printf(" Coarsening Type = PMIS \n");
       }
-      else if (abs(coarsen_type) == 10) 
+      else if (hypre_abs(coarsen_type) == 10) 
       {
 	hypre_printf(" Coarsening Type = HMIS \n");
       }
-      else if (abs(coarsen_type) == 11) 
+      else if (hypre_abs(coarsen_type) == 11) 
       {
 	hypre_printf(" Coarsening Type = Ruge 1st pass only \n");
       }
-      else if (abs(coarsen_type) == 9) 
+      else if (hypre_abs(coarsen_type) == 9) 
       {
 	hypre_printf(" Coarsening Type = PMIS fixed random \n");
       }
-      else if (abs(coarsen_type) == 7) 
+      else if (hypre_abs(coarsen_type) == 7) 
       {
 	hypre_printf(" Coarsening Type = CLJP, fixed random \n");
       }
-      else if (abs(coarsen_type) == 21) /* BM Aug 29, 2006 */
+      else if (hypre_abs(coarsen_type) == 21) /* BM Aug 29, 2006 */
       {
         hypre_printf(" Coarsening Type = CGC \n");
       }
-      else if (abs(coarsen_type) == 22) /* BM Aug 29, 2006 */
+      else if (hypre_abs(coarsen_type) == 22) /* BM Aug 29, 2006 */
       {
         hypre_printf(" Coarsening Type = CGC-E \n");
       }
@@ -931,10 +935,8 @@ hypre_BoomerAMGSetupStats( void               *amg_vdata,
          hypre_printf( "            Number of sweeps:         %4d   %2d  %4d \n",
               num_grid_sweeps[1],
               num_grid_sweeps[2],(2*num_grid_sweeps[3]));
-         if (grid_relax_type[1] == 0)
-            hypre_printf( "   Type 0=Jac, 3=hGS, 6=hSGS, 9=GE:      0    0     0 \n");
-         else
-            hypre_printf( "   Type 0=Jac, 3=hGS, 6=hSGS, 9=GE:     18   18    18 \n");
+         hypre_printf( "   Type 0=Jac, 3=hGS, 6=hSGS, 9=GE:    %2d   %2d   %2d \n", add_rlx, add_rlx, add_rlx);
+         if (add_rlx == 0) hypre_printf( "   Relaxation Weight:   %e \n", add_rlx_wt);
          hypre_printf( "   Point types, partial sweeps (1=C, -1=F):\n");
          hypre_printf( "                  Pre-CG relaxation (down):");
          for (j = 0; j < num_grid_sweeps[1]; j++)
@@ -1004,10 +1006,8 @@ hypre_BoomerAMGSetupStats( void               *amg_vdata,
          hypre_printf( "            Number of sweeps:         %4d   %2d  %4d \n",
               num_grid_sweeps[1],
               num_grid_sweeps[2],(2*num_grid_sweeps[3]));
-         if (grid_relax_type[1] == 0)
-            hypre_printf( "   Type 0=Jac, 3=hGS, 6=hSGS, 9=GE:      0    0     0 \n");
-         else
-            hypre_printf( "   Type 0=Jac, 3=hGS, 6=hSGS, 9=GE:     18   18    18 \n");
+         hypre_printf( "   Type 0=Jac, 3=hGS, 6=hSGS, 9=GE:    %2d   %2d   %2d \n", add_rlx, add_rlx, add_rlx);
+         if (add_rlx == 0) hypre_printf( "   Relaxation Weight:   %e \n", add_rlx_wt);
          hypre_printf( "   Point types, partial sweeps (1=C, -1=F):\n");
          hypre_printf( "                  Pre-CG relaxation (down):");
          for (j = 0; j < num_grid_sweeps[1]; j++)
diff --git a/src/parcsr_ls/par_strength.c b/src/parcsr_ls/par_strength.c
index d85362c..4bb3214 100644
--- a/src/parcsr_ls/par_strength.c
+++ b/src/parcsr_ls/par_strength.c
@@ -1507,7 +1507,7 @@ HYPRE_Int hypre_BoomerAMGCreate2ndS( hypre_ParCSRMatrix *S, HYPRE_Int *CF_marker
       for (i=0 ; i < S_ext_offd_size; i++)
          S_ext_offd_j[i] = hypre_UnorderedIntMapGet(&col_map_offd_C_inverse, S_ext_offd_j[i]);
 
-      hypre_UnorderedIntMapDestroy(&col_map_offd_C_inverse);
+      if (num_cols_offd_C) hypre_UnorderedIntMapDestroy(&col_map_offd_C_inverse);
 #else /* !HYPRE_CONCURRENT_HOPSCOTCH */
       HYPRE_Int cnt_offd, cnt_diag, cnt, value;
       S_ext_diag_size = 0;
diff --git a/src/parcsr_ls/schwarz.c b/src/parcsr_ls/schwarz.c
index 42e1e36..9780f8f 100644
--- a/src/parcsr_ls/schwarz.c
+++ b/src/parcsr_ls/schwarz.c
@@ -1750,7 +1750,7 @@ hypre_AMGCreateDomainDof(hypre_CSRMatrix     *A,
   HYPRE_Int *i_domain_dof, *j_domain_dof;
   HYPRE_Real *domain_matrixinverse;
   HYPRE_Int num_domains;
-  hypre_CSRMatrix *domain_structure;  
+  hypre_CSRMatrix *domain_structure = NULL;  
 
   HYPRE_Int *i_dof_dof = hypre_CSRMatrixI(A);
   HYPRE_Int *j_dof_dof = hypre_CSRMatrixJ(A);
@@ -1785,6 +1785,7 @@ hypre_AMGCreateDomainDof(hypre_CSRMatrix     *A,
   HYPRE_Int cnt;
 
 
+
   /* --------------------------------------------------------------------- */
 
   /*=======================================================================*/
@@ -1794,6 +1795,14 @@ hypre_AMGCreateDomainDof(hypre_CSRMatrix     *A,
   /*hypre_printf("----------- create artificials domain by agglomeration;  ======\n");
 */
 
+  if (num_dofs == 0)
+  {
+     *domain_structure_pointer = domain_structure;
+
+     *piv_pointer = piv;
+
+     return hypre_error_flag;
+  }
 
   i_aggregate_dof = hypre_CTAlloc(HYPRE_Int,num_dofs+1);
   j_aggregate_dof= hypre_CTAlloc(HYPRE_Int,num_dofs);
@@ -3458,7 +3467,7 @@ hypre_ParAMGCreateDomainDof(hypre_ParCSRMatrix   *A,
                             HYPRE_Int **piv_pointer, HYPRE_Int use_nonsymm)
 
 {
-  hypre_CSRMatrix *domain_structure;
+  hypre_CSRMatrix *domain_structure = NULL;
   HYPRE_Int *i_domain_dof, *j_domain_dof;
   HYPRE_Real *domain_matrixinverse;
   HYPRE_Int num_domains;
@@ -3522,6 +3531,15 @@ hypre_ParAMGCreateDomainDof(hypre_ParCSRMatrix   *A,
 
 
 
+  if (num_variables == 0)
+  {
+     *domain_structure_pointer = domain_structure;
+
+     *piv_pointer = piv;
+
+     return hypre_error_flag;
+  }
+
 
   hypre_MPI_Comm_size(hypre_ParCSRMatrixComm(A),&num_procs);
   hypre_MPI_Comm_size(hypre_ParCSRMatrixComm(A),&my_id);
diff --git a/src/parcsr_mv/par_csr_matrix.c b/src/parcsr_mv/par_csr_matrix.c
index b674c39..33ba6a6 100644
--- a/src/parcsr_mv/par_csr_matrix.c
+++ b/src/parcsr_mv/par_csr_matrix.c
@@ -1686,7 +1686,7 @@ hypre_ParCSRMatrixToCSRMatrixAll(hypre_ParCSRMatrix *par_matrix)
 
       /* don't send to myself  - these are sorted so my id would be first*/
       start = 0;
-      if (used_procs[0] == 0)
+      if (num_types && used_procs[0] == 0)
       {
          start = 1;
       }
diff --git a/src/seq_mv/csr_matvec.c b/src/seq_mv/csr_matvec.c
index 44345a0..25c231c 100644
--- a/src/seq_mv/csr_matvec.c
+++ b/src/seq_mv/csr_matvec.c
@@ -105,7 +105,7 @@ hypre_CSRMatrixMatvecOutOfPlace( HYPRE_Complex    alpha,
 #pragma omp parallel for private(i) HYPRE_SMP_SCHEDULE
 #endif
       for (i = 0; i < num_rows*num_vectors; i++)
-         y_data[i] *= beta;
+         y_data[i] = beta*b_data[i];
 
 #ifdef HYPRE_PROFILE
       hypre_profile_times[HYPRE_TIMER_ID_MATVEC] += hypre_MPI_Wtime() - time_begin;
diff --git a/src/seq_mv/vector.c b/src/seq_mv/vector.c
index 0c8c7c0..619ddba 100644
--- a/src/seq_mv/vector.c
+++ b/src/seq_mv/vector.c
@@ -88,7 +88,7 @@ hypre_SeqVectorInitialize( hypre_Vector *vector )
    HYPRE_Int  multivec_storage_method = hypre_VectorMultiVecStorageMethod(vector);
 
    if ( ! hypre_VectorData(vector) )
-      hypre_VectorData(vector) = hypre_TAlloc(HYPRE_Complex, num_vectors*size);
+      hypre_VectorData(vector) = hypre_CTAlloc(HYPRE_Complex, num_vectors*size);
 
    if ( multivec_storage_method == 0 )
    {
diff --git a/src/sstruct_ls/fac_CFInterfaceExtents.c b/src/sstruct_ls/fac_CFInterfaceExtents.c
index c2a6baa..6d251d8 100644
--- a/src/sstruct_ls/fac_CFInterfaceExtents.c
+++ b/src/sstruct_ls/fac_CFInterfaceExtents.c
@@ -19,7 +19,7 @@
    ii = hypre_IndexX(stencil);\
    jj = hypre_IndexY(stencil);\
    kk = hypre_IndexZ(stencil);\
-   abs_shape= abs(ii) + abs(jj) + abs(kk); \
+   abs_shape= hypre_abs(ii) + hypre_abs(jj) + hypre_abs(kk); \
 }
 
 /*--------------------------------------------------------------------------
diff --git a/src/sstruct_ls/fac_amr_fcoarsen.c b/src/sstruct_ls/fac_amr_fcoarsen.c
index 68c7476..1f0299c 100644
--- a/src/sstruct_ls/fac_amr_fcoarsen.c
+++ b/src/sstruct_ls/fac_amr_fcoarsen.c
@@ -59,7 +59,7 @@
       ii = hypre_IndexX(stencil);               \
       jj = hypre_IndexY(stencil);               \
       kk = hypre_IndexZ(stencil);               \
-      abs_shape= abs(ii) + abs(jj) + abs(kk);   \
+      abs_shape= hypre_abs(ii) + hypre_abs(jj) + hypre_abs(kk);   \
    }
 
 /*--------------------------------------------------------------------------
@@ -2963,7 +2963,7 @@ hypre_AMR_FCoarsen( hypre_SStructMatrix  *   A,
                /*-----------------------------------------------------------------
                 * Compute the weights for the averaged stencil contributions.
                 * We need to convert the ranks back to stencil_shapes and then
-                * fine the abs of the stencil shape.
+                * find the abs of the stencil shape.
                 *-----------------------------------------------------------------*/
                temp3= hypre_TAlloc(HYPRE_Real, coarse_stencil_cnt[i]);
                for (j=0; j< coarse_stencil_cnt[i]; j++)
diff --git a/src/sstruct_ls/fac_amr_zero_data.c b/src/sstruct_ls/fac_amr_zero_data.c
index ae36389..5bd1381 100644
--- a/src/sstruct_ls/fac_amr_zero_data.c
+++ b/src/sstruct_ls/fac_amr_zero_data.c
@@ -276,9 +276,9 @@ hypre_ZeroAMRMatrixData(hypre_SStructMatrix  *A,
 
                  for (j= 0; j< stencil_size; j++)
                  {
-                    rank= abs(hypre_IndexX(stencil_shape[j]))+
-                          abs(hypre_IndexY(stencil_shape[j]))+
-                          abs(hypre_IndexZ(stencil_shape[j]));
+                    rank= hypre_abs(hypre_IndexX(stencil_shape[j]))+
+                          hypre_abs(hypre_IndexY(stencil_shape[j]))+
+                          hypre_abs(hypre_IndexZ(stencil_shape[j]));
                    
                     if (rank)
                     {
diff --git a/src/sstruct_ls/fac_cf_coarsen.c b/src/sstruct_ls/fac_cf_coarsen.c
index ebc18a8..21ef981 100644
--- a/src/sstruct_ls/fac_cf_coarsen.c
+++ b/src/sstruct_ls/fac_cf_coarsen.c
@@ -58,7 +58,7 @@
       ii = hypre_IndexX(stencil);               \
       jj = hypre_IndexY(stencil);               \
       kk = hypre_IndexZ(stencil);               \
-      abs_shape= abs(ii) + abs(jj) + abs(kk);   \
+      abs_shape= hypre_abs(ii) + hypre_abs(jj) + hypre_abs(kk);   \
    }
 
 /*--------------------------------------------------------------------------
diff --git a/src/sstruct_ls/fac_cfstencil_box.c b/src/sstruct_ls/fac_cfstencil_box.c
index e8ef56b..3827952 100644
--- a/src/sstruct_ls/fac_cfstencil_box.c
+++ b/src/sstruct_ls/fac_cfstencil_box.c
@@ -19,7 +19,7 @@
    ii = hypre_IndexX(stencil);\
    jj = hypre_IndexY(stencil);\
    kk = hypre_IndexZ(stencil);\
-   abs_shape= abs(ii) + abs(jj) + abs(kk); \
+   abs_shape= hypre_abs(ii) + hypre_abs(jj) + hypre_abs(kk); \
 }
 
 /*--------------------------------------------------------------------------
diff --git a/src/sstruct_ls/fac_zero_stencilcoef.c b/src/sstruct_ls/fac_zero_stencilcoef.c
index 3d54b53..a46d613 100644
--- a/src/sstruct_ls/fac_zero_stencilcoef.c
+++ b/src/sstruct_ls/fac_zero_stencilcoef.c
@@ -19,7 +19,7 @@
       ii = hypre_IndexX(stencil);               \
       jj = hypre_IndexY(stencil);               \
       kk = hypre_IndexZ(stencil);               \
-      abs_shape= abs(ii) + abs(jj) + abs(kk);   \
+      abs_shape= hypre_abs(ii) + hypre_abs(jj) + hypre_abs(kk);   \
    }
 
 /*--------------------------------------------------------------------------
diff --git a/src/sstruct_mv/HYPRE_sstruct_grid.c b/src/sstruct_mv/HYPRE_sstruct_grid.c
index c40fa0f..811415a 100644
--- a/src/sstruct_mv/HYPRE_sstruct_grid.c
+++ b/src/sstruct_mv/HYPRE_sstruct_grid.c
@@ -39,6 +39,7 @@ HYPRE_SStructGridCreate( MPI_Comm           comm,
    HYPRE_Int               *fem_nvars;
    HYPRE_Int              **fem_vars;
    hypre_Index            **fem_offsets;
+   HYPRE_Int                num_ghost[2*HYPRE_MAXDIM];
    HYPRE_Int                i;
 
    grid = hypre_TAlloc(hypre_SStructGrid, 1);
@@ -83,7 +84,14 @@ HYPRE_SStructGridCreate( MPI_Comm           comm,
    hypre_SStructGridRefCount(grid)      = 1;
 
    /* GEC0902 ghost addition to the grid    */
-   hypre_SStructGridGhlocalSize(grid)   = 0; 
+   hypre_SStructGridGhlocalSize(grid)   = 0;
+
+   /* Initialize num ghost */
+   for (i = 0; i < 2*ndim; i++)
+   {
+      num_ghost[i] = 1;
+   }
+   hypre_SStructGridSetNumGhost(grid, num_ghost);
 
    *grid_ptr = grid;
 
@@ -311,7 +319,7 @@ HYPRE_SStructGridSetFEMOrdering( HYPRE_SStructGrid  grid,
          for (d = 0; d < 3; d++)
          {
             loop[d] = 0;
-            if ((varoffset[d] != 0) && (d < ndim))
+            if ((d < ndim) && (varoffset[d] != 0))
             {
                loop[d] = 1;
             }
diff --git a/src/sstruct_mv/_hypre_sstruct_mv.h b/src/sstruct_mv/_hypre_sstruct_mv.h
index 059968d..9586842 100644
--- a/src/sstruct_mv/_hypre_sstruct_mv.h
+++ b/src/sstruct_mv/_hypre_sstruct_mv.h
@@ -181,6 +181,7 @@ typedef struct hypre_SStructGrid_struct
 
    HYPRE_Int               ghlocal_size;  /* GEC0902 Number of vars including ghosts */
    HYPRE_Int               ghstart_rank;  /* GEC0902 start rank including ghosts  */
+   HYPRE_Int               num_ghost[2*HYPRE_MAXDIM]; /* ghost layer size */  
 
 } hypre_SStructGrid;
 
@@ -223,6 +224,7 @@ typedef struct hypre_SStructGrid_struct
 #define hypre_SStructGridRefCount(grid)       ((grid) -> ref_count)
 #define hypre_SStructGridGhlocalSize(grid)    ((grid) -> ghlocal_size)
 #define hypre_SStructGridGhstartRank(grid)    ((grid) -> ghstart_rank)
+#define hypre_SStructGridNumGhost(grid)       ((grid) -> num_ghost)
 
 /*--------------------------------------------------------------------------
  * Accessor macros: hypre_SStructPGrid
diff --git a/src/sstruct_mv/sstruct_grid.c b/src/sstruct_mv/sstruct_grid.c
index a86b9fb..9416c99 100644
--- a/src/sstruct_mv/sstruct_grid.c
+++ b/src/sstruct_mv/sstruct_grid.c
@@ -292,6 +292,7 @@ hypre_SStructPGridAssemble( hypre_SStructPGrid  *pgrid )
       if ((t > 0) && (sgrids[t] == NULL))
       {
          HYPRE_StructGridCreate(comm, ndim, &sgrid);
+         hypre_StructGridSetNumGhost(sgrid, hypre_StructGridNumGhost(cell_sgrid));
          boxes = hypre_BoxArrayCreate(0, ndim);
          hypre_SStructVariableGetOffset((hypre_SStructVariable) t,
                                         ndim, varoffset);
@@ -1916,23 +1917,28 @@ hypre_SStructVarToNborVar( hypre_SStructGrid  *grid,
 HYPRE_Int
 hypre_SStructGridSetNumGhost( hypre_SStructGrid  *grid, HYPRE_Int *num_ghost )
 {
+   HYPRE_Int             ndim   = hypre_SStructGridNDim(grid);
    HYPRE_Int             nparts = hypre_SStructGridNParts(grid);
-   HYPRE_Int             nvars ;
-   HYPRE_Int             part  ;
-   HYPRE_Int             var  ;
-   hypre_SStructPGrid    *pgrid;
-   hypre_StructGrid      *sgrid;
+   HYPRE_Int             part, i, t;
+   hypre_SStructPGrid   *pgrid;
+   hypre_StructGrid     *sgrid;
 
-   for (part = 0; part < nparts; part++)
+   for (i = 0; i < 2*ndim; i++)
    {
+      hypre_SStructGridNumGhost(grid)[i] = num_ghost[i];
+   }
 
+   for (part = 0; part < nparts; part++)
+   {
       pgrid = hypre_SStructGridPGrid(grid, part);
-      nvars = hypre_SStructPGridNVars(pgrid);
      
-      for ( var = 0; var < nvars; var++)
+      for (t = 0; t < 8; t++)
       {
-         sgrid = hypre_SStructPGridSGrid(pgrid, var);
-         hypre_StructGridSetNumGhost(sgrid, num_ghost);
+         sgrid = hypre_SStructPGridVTSGrid(pgrid, t);
+         if (sgrid != NULL)
+         {
+            hypre_StructGridSetNumGhost(sgrid, num_ghost);
+         }
       }
    }
 
diff --git a/src/sstruct_mv/sstruct_grid.h b/src/sstruct_mv/sstruct_grid.h
index f54d52f..4fe9177 100644
--- a/src/sstruct_mv/sstruct_grid.h
+++ b/src/sstruct_mv/sstruct_grid.h
@@ -164,6 +164,7 @@ typedef struct hypre_SStructGrid_struct
 
    HYPRE_Int               ghlocal_size;  /* GEC0902 Number of vars including ghosts */
    HYPRE_Int               ghstart_rank;  /* GEC0902 start rank including ghosts  */
+   HYPRE_Int               num_ghost[2*HYPRE_MAXDIM]; /* ghost layer size */  
 
 } hypre_SStructGrid;
 
@@ -206,6 +207,7 @@ typedef struct hypre_SStructGrid_struct
 #define hypre_SStructGridRefCount(grid)       ((grid) -> ref_count)
 #define hypre_SStructGridGhlocalSize(grid)    ((grid) -> ghlocal_size)
 #define hypre_SStructGridGhstartRank(grid)    ((grid) -> ghstart_rank)
+#define hypre_SStructGridNumGhost(grid)       ((grid) -> num_ghost)
 
 /*--------------------------------------------------------------------------
  * Accessor macros: hypre_SStructPGrid
diff --git a/src/sstruct_mv/sstruct_matrix.c b/src/sstruct_mv/sstruct_matrix.c
index 30d3c7b..c3e221e 100644
--- a/src/sstruct_mv/sstruct_matrix.c
+++ b/src/sstruct_mv/sstruct_matrix.c
@@ -228,19 +228,20 @@ hypre_SStructPMatrixInitialize( hypre_SStructPMatrix *pmatrix )
 {
    HYPRE_Int             nvars        = hypre_SStructPMatrixNVars(pmatrix);
    HYPRE_Int           **symmetric    = hypre_SStructPMatrixSymmetric(pmatrix);
-   HYPRE_Int             num_ghost[2*HYPRE_MAXDIM];
    hypre_StructMatrix   *smatrix;
-   HYPRE_Int             vi, vj, d, ndim;
+   HYPRE_Int             vi, vj;
+   /* HYPRE_Int             num_ghost[2*HYPRE_MAXDIM]; */
+   /* HYPRE_Int             vi, vj, d, ndim; */
 
+#if 0
    ndim = hypre_SStructPMatrixNDim(pmatrix);
+   /* RDF: Why are the ghosts being reset to one? Maybe it needs to be at least
+    * one to set shared coefficients correctly, but not exactly one? */
    for (d = 0; d < ndim; d++)
    {
       num_ghost[2*d] = num_ghost[2*d+1] = 1;
    }
-   for (d = ndim; d < ndim; d++)
-   {
-      num_ghost[2*d] = num_ghost[2*d+1] = 0;
-   }
+#endif
    for (vi = 0; vi < nvars; vi++)
    {
       for (vj = 0; vj < nvars; vj++)
@@ -249,7 +250,7 @@ hypre_SStructPMatrixInitialize( hypre_SStructPMatrix *pmatrix )
          if (smatrix != NULL)
          {
             HYPRE_StructMatrixSetSymmetric(smatrix, symmetric[vi][vj]);
-            hypre_StructMatrixSetNumGhost(smatrix, num_ghost);
+            /* hypre_StructMatrixSetNumGhost(smatrix, num_ghost); */
             hypre_StructMatrixInitialize(smatrix);
             /* needed to get AddTo accumulation correct between processors */
             hypre_StructMatrixClearGhostValues(smatrix);
@@ -505,10 +506,6 @@ hypre_SStructPMatrixAccumulate( hypre_SStructPMatrix *pmatrix )
       return hypre_error_flag;
    }
 
-   for (d = ndim; d < ndim; d++)
-   {
-      num_ghost[2*d] = num_ghost[2*d+1] = 0;
-   }
    for (vi = 0; vi < nvars; vi++)
    {
       for (vj = 0; vj < nvars; vj++)
@@ -1125,6 +1122,7 @@ hypre_SStructUMatrixSetBoxValues( hypre_SStructMatrix *matrix,
       hypre_BoxLoop0For()
       {
          hypre_BoxLoopGetIndex(index);
+         hypre_AddIndexes(index, hypre_BoxIMin(vbox), ndim, index);
          hypre_SStructUMatrixSetValues(matrix, part, index, var,
                                        nentries, entries, values, action);
          values += nentries;
diff --git a/src/sstruct_mv/sstruct_vector.c b/src/sstruct_mv/sstruct_vector.c
index e7c2012..939c7f8 100644
--- a/src/sstruct_mv/sstruct_vector.c
+++ b/src/sstruct_mv/sstruct_vector.c
@@ -360,10 +360,6 @@ hypre_SStructPVectorAccumulate( hypre_SStructPVector *pvector )
       return hypre_error_flag;
    }
 
-   for (d = ndim; d < ndim; d++)
-   {
-      num_ghost[2*d] = num_ghost[2*d+1] = 0;
-   }
    for (var = 0; var < nvars; var++)
    {
       if (vartypes[var] > 0)
diff --git a/src/struct_ls/Makefile b/src/struct_ls/Makefile
index c8b187c..4653b73 100644
--- a/src/struct_ls/Makefile
+++ b/src/struct_ls/Makefile
@@ -14,7 +14,7 @@
 include ../config/Makefile.config
 
 CINCLUDES = ${INCLUDES} ${MPIINCLUDE}
- 
+
 C_COMPILE_FLAGS = \
  -I..\
  -I$(srcdir)\
diff --git a/src/struct_ls/pfmg.c b/src/struct_ls/pfmg.c
index 4e0efca..99a4829 100644
--- a/src/struct_ls/pfmg.c
+++ b/src/struct_ls/pfmg.c
@@ -60,6 +60,8 @@ hypre_PFMGDestroy( void *pfmg_vdata )
 	hypre_PFMGData *pfmg_data = (hypre_PFMGData *)pfmg_vdata;
 
    HYPRE_Int l;
+   
+   HYPRE_ANNOTATION_BEGIN("PFMG.destroy");
 
    if (pfmg_data)
    {
@@ -121,6 +123,8 @@ hypre_PFMGDestroy( void *pfmg_vdata )
       hypre_TFree(pfmg_data);
    }
 
+   HYPRE_ANNOTATION_END("PFMG.destroy");
+
    return hypre_error_flag;
 }
 
diff --git a/src/struct_ls/pfmg_setup.c b/src/struct_ls/pfmg_setup.c
index 0f64a24..b032c0c 100644
--- a/src/struct_ls/pfmg_setup.c
+++ b/src/struct_ls/pfmg_setup.c
@@ -108,6 +108,7 @@ hypre_PFMGSetup( void               *pfmg_vdata,
    char                  filename[255];
 #endif
 
+   HYPRE_ANNOTATION_BEGIN("PFMG.setup");
 
    /*-----------------------------------------------------
     * Set up coarse grids
@@ -581,6 +582,8 @@ hypre_PFMGSetup( void               *pfmg_vdata,
    hypre_StructMatrixPrint(filename, A_l[l], 0);
 #endif
 
+   HYPRE_ANNOTATION_END("PFMG.setup");
+
    return hypre_error_flag;
 }
 
diff --git a/src/struct_ls/pfmg_solve.c b/src/struct_ls/pfmg_solve.c
index a35c0fc..88fd973 100644
--- a/src/struct_ls/pfmg_solve.c
+++ b/src/struct_ls/pfmg_solve.c
@@ -77,6 +77,8 @@ hypre_PFMGSolve( void               *pfmg_vdata,
     * Initialize some things and deal with special cases
     *-----------------------------------------------------*/
 
+   HYPRE_ANNOTATION_BEGIN("PFMG.solve");
+
    hypre_BeginTiming(pfmg_data -> time_index);
 
    constant_coefficient = hypre_StructMatrixConstantCoefficient(A);
@@ -100,6 +102,8 @@ hypre_PFMGSolve( void               *pfmg_vdata,
       }
 
       hypre_EndTiming(pfmg_data -> time_index);
+      HYPRE_ANNOTATION_END("PFMG.solve");
+
       return hypre_error_flag;
    }
 
@@ -121,6 +125,8 @@ hypre_PFMGSolve( void               *pfmg_vdata,
          }
 
          hypre_EndTiming(pfmg_data -> time_index);
+         HYPRE_ANNOTATION_END("PFMG.solve");
+
          return hypre_error_flag;
       }
    }
@@ -322,6 +328,7 @@ hypre_PFMGSolve( void               *pfmg_vdata,
    }
 
    hypre_EndTiming(pfmg_data -> time_index);
+   HYPRE_ANNOTATION_END("PFMG.solve");
 
    return hypre_error_flag;
 }
diff --git a/src/struct_ls/red_black_gs.c b/src/struct_ls/red_black_gs.c
index 25986da..4cd8b32 100644
--- a/src/struct_ls/red_black_gs.c
+++ b/src/struct_ls/red_black_gs.c
@@ -19,10 +19,6 @@
 #include "_hypre_struct_ls.h"
 #include "red_black_gs.h"
 
-#ifndef hypre_abs
-#define hypre_abs(a)  (((a)>0) ? (a) : -(a))
-#endif
-
 /*--------------------------------------------------------------------------
  *--------------------------------------------------------------------------*/
 
diff --git a/src/struct_ls/smg.c b/src/struct_ls/smg.c
index 89637ec..3a10636 100644
--- a/src/struct_ls/smg.c
+++ b/src/struct_ls/smg.c
@@ -57,6 +57,8 @@ hypre_SMGDestroy( void *smg_vdata )
 
    HYPRE_Int l;
 
+   HYPRE_ANNOTATION_BEGIN("SMG.destroy");
+
    if (smg_data)
    {
       if ((smg_data -> logging) > 0)
@@ -124,6 +126,8 @@ hypre_SMGDestroy( void *smg_vdata )
       hypre_FinalizeTiming(smg_data -> time_index);
       hypre_TFree(smg_data);
    }
+   
+   HYPRE_ANNOTATION_END("SMG.destroy");
 
    return hypre_error_flag;
 }
diff --git a/src/struct_ls/smg_setup.c b/src/struct_ls/smg_setup.c
index 5f69c65..13a753d 100644
--- a/src/struct_ls/smg_setup.c
+++ b/src/struct_ls/smg_setup.c
@@ -89,6 +89,8 @@ hypre_SMGSetup( void               *smg_vdata,
     * Set up coarsening direction
     *-----------------------------------------------------*/
 
+   HYPRE_ANNOTATION_BEGIN("SMG.setup");
+
    cdir = hypre_StructStencilNDim(hypre_StructMatrixStencil(A)) - 1;
    (smg_data -> cdir) = cdir;
 
@@ -427,6 +429,8 @@ hypre_SMGSetup( void               *smg_vdata,
    }
 #endif
 
+   HYPRE_ANNOTATION_END("SMG.setup");
+
    return hypre_error_flag;
 }
 
diff --git a/src/struct_ls/smg_solve.c b/src/struct_ls/smg_solve.c
index 95a8d59..2a36fe7 100644
--- a/src/struct_ls/smg_solve.c
+++ b/src/struct_ls/smg_solve.c
@@ -93,6 +93,7 @@ hypre_SMGSolve( void               *smg_vdata,
     * Initialize some things and deal with special cases
     *-----------------------------------------------------*/
 
+   HYPRE_ANNOTATION_BEGIN("SMG.solve");
    hypre_BeginTiming(smg_data -> time_index);
 
    hypre_StructMatrixDestroy(A_l[0]);
@@ -114,6 +115,8 @@ hypre_SMGSolve( void               *smg_vdata,
       }
 
       hypre_EndTiming(smg_data -> time_index);
+      HYPRE_ANNOTATION_END("SMG.solve");
+
       return hypre_error_flag;
    }
 
@@ -135,6 +138,8 @@ hypre_SMGSolve( void               *smg_vdata,
          }
 
          hypre_EndTiming(smg_data -> time_index);
+         HYPRE_ANNOTATION_END("SMG.solve");
+
          return hypre_error_flag;
       }
    }
@@ -319,6 +324,7 @@ hypre_SMGSolve( void               *smg_vdata,
    }
 
    hypre_EndTiming(smg_data -> time_index);
+   HYPRE_ANNOTATION_END("SMG.solve");
 
    return hypre_error_flag;
 }
diff --git a/src/struct_mv/assumed_part.c b/src/struct_mv/assumed_part.c
index a6d031f..a3be50f 100644
--- a/src/struct_mv/assumed_part.c
+++ b/src/struct_mv/assumed_part.c
@@ -772,7 +772,7 @@ hypre_StructAssumedPartitionCreate(
    HYPRE_Int   proc_alloc, count, box_count;
    HYPRE_Int   max_response_size;
    HYPRE_Int  *response_buf = NULL, *response_buf_starts=NULL;
-   HYPRE_Int  *tmp_box_nums = NULL, *tmp_proc_ids = NULL;
+   HYPRE_Int  *tmp_proc_ids = NULL, *tmp_box_nums = NULL, *tmp_box_inds = NULL;
    HYPRE_Int  *proc_array_starts=NULL;
 
    hypre_BoxArray              *my_partition;
@@ -1109,7 +1109,7 @@ hypre_StructAssumedPartitionCreate(
       }
       else
       {
-         while (proc_array[i] <=1 && i < size) /* size is the number of regions */
+         while (i < size && proc_array[i] <=1) /* size is the number of regions */
          {
             i++;
          }
@@ -1255,8 +1255,10 @@ hypre_StructAssumedPartitionCreate(
    /* Don't want to allocate too much memory here */
    size = 1.2 * hypre_BoxArraySize(local_boxes);
 
-   tmp_box_nums = hypre_CTAlloc(HYPRE_Int, size);
-   tmp_proc_ids =  hypre_CTAlloc(HYPRE_Int, size);
+   /* Each local box may live on multiple procs in the assumed partition */
+   tmp_proc_ids = hypre_CTAlloc(HYPRE_Int, size); /* local box proc ids */
+   tmp_box_nums = hypre_CTAlloc(HYPRE_Int, size); /* local box boxnum */
+   tmp_box_inds = hypre_CTAlloc(HYPRE_Int, size); /* local box array index */
 
    proc_count = 0;
    count = 0; /* Current number of procs */
@@ -1273,13 +1275,15 @@ hypre_StructAssumedPartitionCreate(
       {
          size = size + proc_count + 1.2*(hypre_BoxArraySize(local_boxes)-i);
          /* hypre_printf("myid = %d, *adjust* alloc size = %d\n", myid, size);*/
-         tmp_box_nums = hypre_TReAlloc(tmp_box_nums, HYPRE_Int, size);
          tmp_proc_ids = hypre_TReAlloc(tmp_proc_ids, HYPRE_Int, size);
+         tmp_box_nums = hypre_TReAlloc(tmp_box_nums, HYPRE_Int, size);
+         tmp_box_inds = hypre_TReAlloc(tmp_box_inds, HYPRE_Int, size);
       }
       for (j = 0; j < proc_count; j++)
       {
-         tmp_box_nums[count] = local_boxnums[i];
          tmp_proc_ids[count] = proc_array[j];
+         tmp_box_nums[count] = local_boxnums[i];
+         tmp_box_inds[count] = i;
          count++;
       }
    }
@@ -1291,7 +1295,7 @@ hypre_StructAssumedPartitionCreate(
       and then create a new buffer to send to the exchange data function. */
 
    /* Sort the proc_ids */
-   hypre_qsort2i(tmp_proc_ids, tmp_box_nums, 0, count-1);
+   hypre_qsort3i(tmp_proc_ids, tmp_box_nums, tmp_box_inds, 0, count-1);
      
    /* Use proc_array for the processor ids to contact.  Use box array to get our
       boxes and then pass the array only (not the structure) to exchange data. */
@@ -1311,7 +1315,7 @@ hypre_StructAssumedPartitionCreate(
       proc_array[0] = tmp_proc_ids[0];
 
       contact_boxinfo[index++] = tmp_box_nums[0];
-      box = hypre_BoxArrayBox(local_boxes, tmp_box_nums[0]);
+      box = hypre_BoxArrayBox(local_boxes, tmp_box_inds[0]);
       for (d = 0; d < ndim; d++)
       {
          contact_boxinfo[index++] = hypre_BoxIMinD(box, d);
@@ -1332,7 +1336,7 @@ hypre_StructAssumedPartitionCreate(
       /* These boxes are not copied in a particular order */
         
       contact_boxinfo[index++] = tmp_box_nums[i];
-      box = hypre_BoxArrayBox(local_boxes, tmp_box_nums[i]);
+      box = hypre_BoxArrayBox(local_boxes, tmp_box_inds[i]);
       for (d = 0; d < ndim; d++)
       {
          contact_boxinfo[index++] = hypre_BoxIMinD(box, d);
@@ -1344,6 +1348,7 @@ hypre_StructAssumedPartitionCreate(
    /* Clean up */
    hypre_TFree(tmp_proc_ids);
    hypre_TFree(tmp_box_nums);
+   hypre_TFree(tmp_box_inds);
 
    /* EXCHANGE DATA */
 
diff --git a/src/struct_mv/struct_grid.c b/src/struct_mv/struct_grid.c
index deffd18..8a1fe6a 100644
--- a/src/struct_mv/struct_grid.c
+++ b/src/struct_mv/struct_grid.c
@@ -62,12 +62,7 @@ hypre_StructGridCreate( MPI_Comm           comm,
    hypre_StructGridGhlocalSize(grid)  = 0;
    for (i = 0; i < 2*ndim; i++)
    {
-      hypre_StructGridNumGhost(grid)[i] = 0;
-   }
-   for (i = 0; i < ndim; i++)
-   {
-      hypre_StructGridNumGhost(grid)[2*i] = 1;
-      hypre_StructGridNumGhost(grid)[2*i+1] = 1;
+      hypre_StructGridNumGhost(grid)[i] = 1;
    }
 
    *grid_ptr = grid;
diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt
index 187753b..eef13d0 100644
--- a/src/test/CMakeLists.txt
+++ b/src/test/CMakeLists.txt
@@ -55,9 +55,6 @@ endif ()
 add_executable (ij ij.c)
 target_link_libraries (ij ${HYPRE_LIBS})
  
-add_executable (new_ij new_ij.c)
-target_link_libraries (new_ij ${HYPRE_LIBS})
- 
 add_executable (sstruct sstruct.c)
 target_link_libraries (sstruct ${HYPRE_LIBS})
  
diff --git a/src/test/Makefile b/src/test/Makefile
index 2670d5a..9ecd6ab 100644
--- a/src/test/Makefile
+++ b/src/test/Makefile
@@ -13,7 +13,7 @@
 
 include ../config/Makefile.config
 
-CINCLUDES = ${MPIINCLUDE}
+CINCLUDES = ${INCLUDES} ${MPIINCLUDE}
 CDEFS = -DHYPRE_TIMING -DHYPRE_FORTRAN
 CXXDEFS = -DNOFEI -DHYPRE_TIMING -DMPICH_SKIP_MPICXX
 
@@ -57,7 +57,6 @@ LFLAGS =\
 
 HYPRE_DRIVERS =\
  ij.c\
- new_ij.c\
  sstruct.c\
  struct.c\
  ams_driver.c\
@@ -129,81 +128,77 @@ distclean: clean
 
 ij: ij.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ $@.o ${CFLAGS} ${LFLAGS}
-
-new_ij: new_ij.o
-	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ $@.o ${CFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ $@.o ${LFLAGS}
 
 sstruct: sstruct.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ $@.o ${CFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ $@.o ${LFLAGS}
 
 struct: struct.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ $@.o ${CFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ $@.o ${LFLAGS}
 
 ams_driver: ams_driver.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ $@.o ${CFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ $@.o ${LFLAGS}
 
 maxwell_unscaled: maxwell_unscaled.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ $@.o ${CFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ $@.o ${LFLAGS}
 
 struct_migrate: struct_migrate.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ $@.o ${CFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ $@.o ${LFLAGS}
 
 sstruct_fac: sstruct_fac.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ $@.o ${CFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ $@.o ${LFLAGS}
 
 ij_mv: ij_mv.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ $@.o ${CFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ $@.o ${LFLAGS}
 
 zboxloop: zboxloop.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ $@.o ${CFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ $@.o ${LFLAGS}
 
 # RDF: Keep these for now
 
 hypre_set_precond: hypre_set_precond.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ $@.o ${CFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ $@.o ${LFLAGS}
 
 test_ij: hypre_set_precond.o test_ij.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ hypre_set_precond.o $@.o ${CFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ hypre_set_precond.o $@.o ${LFLAGS}
 
 driver_commpkg: driver_commpkg.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ $@.o ${CFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ $@.o ${LFLAGS}
 
 # C++
 
 fei: fei.o
 	@echo  "Building" $@ "... "
-	${LINK_CXX} -o $@ $@.o ${CXXFLAGS} ${LFLAGS}
+	${LINK_CXX} -o $@ $@.o ${LFLAGS}
 
 cxx_ij: cxx_ij.o
 	@echo  "Building" $@ "... "
-	${LINK_CXX} -o $@ $@.o ${CXXFLAGS} ${LFLAGS}
+	${LINK_CXX} -o $@ $@.o ${LFLAGS}
 cxx_ij.o: cxx_ij.cxx
 cxx_ij.cxx: ij.c
 	cp -fp ij.c cxx_ij.cxx
 
 cxx_sstruct: cxx_sstruct.o
 	@echo  "Building" $@ "... "
-	${LINK_CXX} -o $@ $@.o ${CXXFLAGS} ${LFLAGS}
+	${LINK_CXX} -o $@ $@.o ${LFLAGS}
 cxx_sstruct.o: cxx_sstruct.cxx
 cxx_sstruct.cxx: sstruct.c
 	cp -fp sstruct.c cxx_sstruct.cxx
 
 cxx_struct: cxx_struct.o
 	@echo  "Building" $@ "... "
-	${LINK_CXX} -o $@ $@.o ${CXXFLAGS} ${LFLAGS}
+	${LINK_CXX} -o $@ $@.o ${LFLAGS}
 cxx_struct.o: cxx_struct.cxx
 cxx_struct.cxx: struct.c
 	cp -fp struct.c cxx_struct.cxx
@@ -212,71 +207,71 @@ cxx_struct.cxx: struct.c
 
 f77_ij: f77_ij.o
 	@echo  "Building" $@ "... "
-	${LINK_FC} -o $@ $@.o ${FFLAGS} ${LFLAGS}
+	${LINK_FC} -o $@ $@.o ${LFLAGS}
 
 f77_ij_mv: f77_ij_mv.o
 	@echo  "Building" $@ "... "
-	${LINK_FC} -o $@ $@.o ${FFLAGS} ${LFLAGS}
+	${LINK_FC} -o $@ $@.o ${LFLAGS}
 
 f77_struct: f77_struct.o
 	@echo  "Building" $@ "... "
-	${LINK_FC} -o $@ $@.o ${FFLAGS} ${LFLAGS}
+	${LINK_FC} -o $@ $@.o ${LFLAGS}
 
 # RDF: Keep these for now
 
 struct_for: fstruct_mv.o fstruct_ls.o struct_for.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ fstruct_mv.o fstruct_ls.o $@.o ${FFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ fstruct_mv.o fstruct_ls.o $@.o ${LFLAGS}
 
 ex1_for: fstruct_mv.o fstruct_ls.o ex1_for.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ fstruct_mv.o fstruct_ls.o $@.o ${FFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ fstruct_mv.o fstruct_ls.o $@.o ${LFLAGS}
 
 ex3_for: fstruct_mv.o fstruct_ls.o ex3_for.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ fstruct_mv.o fstruct_ls.o $@.o ${FFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ fstruct_mv.o fstruct_ls.o $@.o ${LFLAGS}
 
 ex5_for: fij_mv.o fparcsr_mv.o fparcsr_ls.o ex5_for.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ fij_mv.o fparcsr_mv.o fparcsr_ls.o $@.o ${FFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ fij_mv.o fparcsr_mv.o fparcsr_ls.o $@.o ${LFLAGS}
 
 ex6_for: fstruct_mv.o fstruct_ls.o fsstruct_mv.o fsstruct_ls.o ex6_for.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ fstruct_mv.o fstruct_ls.o fsstruct_mv.o fsstruct_ls.o $@.o ${FFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ fstruct_mv.o fstruct_ls.o fsstruct_mv.o fsstruct_ls.o $@.o ${LFLAGS}
 
 ex7_for: fstruct_mv.o fstruct_ls.o fsstruct_mv.o fsstruct_ls.o ex7_for.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ fstruct_mv.o fstruct_ls.o fsstruct_mv.o fsstruct_ls.o $@.o ${FFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ fstruct_mv.o fstruct_ls.o fsstruct_mv.o fsstruct_ls.o $@.o ${LFLAGS}
 
 for_maxwell: fparcsr_mv.o fsstruct_mv.o fsstruct_ls.o for_maxwell.o
 	@echo  "Building" $@ "... "
-	${LINK_CC} -o $@ fparcsr_mv.o fsstruct_mv.o fsstruct_ls.o $@.o ${FFLAGS} ${LFLAGS}
+	${LINK_CC} -o $@ fparcsr_mv.o fsstruct_mv.o fsstruct_ls.o $@.o ${LFLAGS}
 
 fij_mv: fij_mv.f
 	@echo  "Building" $@ "... "
-	${LINK_FC} -c $@  ${FFLAGS}
+	${LINK_FC} -c $@
 
 fparcsr_ls: fparcsr_ls.f
 	@echo  "Building" $@ "... "
-	${LINK_FC} -c $@  ${FFLAGS}
+	${LINK_FC} -c $@
 
 fparcsr_mv: fparcsr_mv.f
 	@echo  "Building" $@ "... "
-	${LINK_FC} -c $@  ${FFLAGS}
+	${LINK_FC} -c $@
 
 fsstruct_ls: fsstruct_ls.f 
 	@echo  "Building" $@ "... "
-	${LINK_FC} -c $@  ${FFLAGS}
+	${LINK_FC} -c $@
 
 fsstruct_mv: fsstruct_mv.f
 	@echo  "Building" $@ "... "
-	${LINK_FC} -c $@  ${FFLAGS}
+	${LINK_FC} -c $@
 
 fstruct_ls: fstruct_ls.f
 	@echo  "Building" $@ "... "
-	${LINK_FC} -c $@  ${FFLAGS}
+	${LINK_FC} -c $@
 
 fstruct_mv: fstruct_mv.f
 	@echo  "Building" $@ "... "
-	${LINK_FC} -c $@  ${FFLAGS}
+	${LINK_FC} -c $@
 
diff --git a/src/test/TEST_ams/solvers.saved b/src/test/TEST_ams/solvers.saved
index 236f40f..210570d 100644
--- a/src/test/TEST_ams/solvers.saved
+++ b/src/test/TEST_ams/solvers.saved
@@ -1,117 +1,117 @@
 # Output file: solvers.out.0
-    Cycle 13   1.324516e-04    0.424120     9.864239e-06 
-    Cycle 14   5.620900e-05    0.424374     4.186127e-06 
-    Cycle 15   2.386874e-05    0.424643     1.777608e-06 
-    Cycle 16   1.013924e-05    0.424792     7.551133e-07 
+    Cycle 14   7.391011e-05    0.433319     5.504405e-06 
+    Cycle 15   3.204481e-05    0.433565     2.386515e-06 
+    Cycle 16   1.389710e-05    0.433677     1.034977e-06 
+    Cycle 17   6.028819e-06    0.433818     4.489922e-07 
 
 
- Average Convergence Factor = 0.414358
+ Average Convergence Factor = 0.423255
 
 # Output file: solvers.out.1
-    Cycle 13   1.324516e-04    0.424120     9.864239e-06 
-    Cycle 14   5.620900e-05    0.424374     4.186127e-06 
-    Cycle 15   2.386874e-05    0.424643     1.777608e-06 
-    Cycle 16   1.013924e-05    0.424792     7.551133e-07 
+    Cycle 14   7.391011e-05    0.433319     5.504405e-06 
+    Cycle 15   3.204481e-05    0.433565     2.386515e-06 
+    Cycle 16   1.389710e-05    0.433677     1.034977e-06 
+    Cycle 17   6.028819e-06    0.433818     4.489922e-07 
 
 
- Average Convergence Factor = 0.414358
+ Average Convergence Factor = 0.423255
 
 # Output file: solvers.out.2
-    Cycle 39   3.139684e-05    0.718260     2.338258e-06 
-    Cycle 40   2.255014e-05    0.718230     1.679407e-06 
-    Cycle 41   1.620358e-05    0.718558     1.206751e-06 
-    Cycle 42   1.164287e-05    0.718537     8.670948e-07 
+    Cycle 39   3.556869e-05    0.721136     2.648954e-06 
+    Cycle 40   2.565030e-05    0.721148     1.910288e-06 
+    Cycle 41   1.850064e-05    0.721264     1.377823e-06 
+    Cycle 42   1.334647e-05    0.721406     9.939689e-07 
 
 
- Average Convergence Factor = 0.717246
+ Average Convergence Factor = 0.719582
 
 # Output file: solvers.out.3
-    Cycle 39   3.139684e-05    0.718260     2.338258e-06 
-    Cycle 40   2.255014e-05    0.718230     1.679407e-06 
-    Cycle 41   1.620358e-05    0.718558     1.206751e-06 
-    Cycle 42   1.164287e-05    0.718537     8.670948e-07 
+    Cycle 39   3.556869e-05    0.721136     2.648954e-06 
+    Cycle 40   2.565030e-05    0.721148     1.910288e-06 
+    Cycle 41   1.850064e-05    0.721264     1.377823e-06 
+    Cycle 42   1.334647e-05    0.721406     9.939689e-07 
 
 
- Average Convergence Factor = 0.717246
+ Average Convergence Factor = 0.719582
 
 # Output file: solvers.out.4
 
 Iterations = 6
-Final Relative Residual Norm = 5.708560e-07
+Final Relative Residual Norm = 6.440133e-07
 
 # Output file: solvers.out.5
 
 Iterations = 6
-Final Relative Residual Norm = 5.708560e-07
+Final Relative Residual Norm = 6.440133e-07
 
 # Output file: solvers.out.6
 
 Iterations = 9
-Final Relative Residual Norm = 8.694579e-07
+Final Relative Residual Norm = 9.646786e-07
 
 # Output file: solvers.out.7
 
 Iterations = 9
-Final Relative Residual Norm = 8.694579e-07
+Final Relative Residual Norm = 9.646786e-07
 
 # Output file: solvers.out.12
 
 Iterations = 18
-Final Relative Residual Norm = 3.770284e-03
+Final Relative Residual Norm = 3.850011e-03
 
 # Output file: solvers.out.8
 
-Eigenvalue lambda   3.02357653918370e+01
-Eigenvalue lambda   3.03135374700743e+01
-Eigenvalue lambda   3.85013899426209e+01
-Eigenvalue lambda   5.14395940120878e+01
-Eigenvalue lambda   5.15742481852129e+01
-Residual   5.42817556275287e-05
-Residual   3.82586769223414e-05
-Residual   4.42815440440115e-05
-Residual   1.13246666444288e-04
-Residual   1.45113200892029e-04
-
-16 iterations
+Eigenvalue lambda   3.02357653918384e+01
+Eigenvalue lambda   3.03135374700770e+01
+Eigenvalue lambda   3.85013899426320e+01
+Eigenvalue lambda   5.14395940110704e+01
+Eigenvalue lambda   5.15742481830717e+01
+Residual   5.87463691694547e-05
+Residual   3.93631178482166e-05
+Residual   4.65887563072628e-05
+Residual   1.00557024939354e-04
+Residual   9.03586581004640e-05
+
+17 iterations
 # Output file: solvers.out.9
 
-Eigenvalue lambda   3.02357653918370e+01
-Eigenvalue lambda   3.03135374700743e+01
-Eigenvalue lambda   3.85013899426209e+01
-Eigenvalue lambda   5.14395940120878e+01
-Eigenvalue lambda   5.15742481852129e+01
-Residual   5.42817556275287e-05
-Residual   3.82586769223414e-05
-Residual   4.42815440440115e-05
-Residual   1.13246666444288e-04
-Residual   1.45113200892029e-04
-
-16 iterations
+Eigenvalue lambda   3.02357653918384e+01
+Eigenvalue lambda   3.03135374700770e+01
+Eigenvalue lambda   3.85013899426320e+01
+Eigenvalue lambda   5.14395940110704e+01
+Eigenvalue lambda   5.15742481830717e+01
+Residual   5.87463691694547e-05
+Residual   3.93631178482166e-05
+Residual   4.65887563072628e-05
+Residual   1.00557024939354e-04
+Residual   9.03586581004640e-05
+
+17 iterations
 # Output file: solvers.out.10
 
-Eigenvalue lambda   3.02357653921602e+01
-Eigenvalue lambda   3.03135374704467e+01
-Eigenvalue lambda   3.85013899427731e+01
-Eigenvalue lambda   5.14395940120470e+01
-Eigenvalue lambda   5.15742481835141e+01
-Residual   1.05389987236677e-04
-Residual   1.12079846702052e-04
-Residual   7.31910174235653e-05
-Residual   1.08463214512112e-04
-Residual   1.01516873550267e-04
+Eigenvalue lambda   3.02357653921933e+01
+Eigenvalue lambda   3.03135374704450e+01
+Eigenvalue lambda   3.85013899428099e+01
+Eigenvalue lambda   5.14395940120781e+01
+Eigenvalue lambda   5.15742481835172e+01
+Residual   1.12876127062766e-04
+Residual   1.14382477254247e-04
+Residual   8.18301203762700e-05
+Residual   1.11751810983326e-04
+Residual   9.84441562127637e-05
 
 23 iterations
 # Output file: solvers.out.11
 
-Eigenvalue lambda   3.02357653921602e+01
-Eigenvalue lambda   3.03135374704467e+01
-Eigenvalue lambda   3.85013899427731e+01
-Eigenvalue lambda   5.14395940120470e+01
-Eigenvalue lambda   5.15742481835141e+01
-Residual   1.05389987236677e-04
-Residual   1.12079846702052e-04
-Residual   7.31910174235653e-05
-Residual   1.08463214512112e-04
-Residual   1.01516873550267e-04
+Eigenvalue lambda   3.02357653921933e+01
+Eigenvalue lambda   3.03135374704450e+01
+Eigenvalue lambda   3.85013899428099e+01
+Eigenvalue lambda   5.14395940120781e+01
+Eigenvalue lambda   5.15742481835172e+01
+Residual   1.12876127062766e-04
+Residual   1.14382477254247e-04
+Residual   8.18301203762700e-05
+Residual   1.11751810983326e-04
+Residual   9.84441562127637e-05
 
 23 iterations
diff --git a/src/test/TEST_examples/bigint.saved b/src/test/TEST_examples/bigint.saved
index f0fadcc..e7aa7b8 100644
--- a/src/test/TEST_examples/bigint.saved
+++ b/src/test/TEST_examples/bigint.saved
@@ -99,7 +99,7 @@ SStruct phase times:
 SStruct Setup:
   wall clock time = 0.210000 seconds
   wall MFLOPS     = 0.000000
-  cpu clock time  = 0.200000 seconds
+  cpu clock time  = 0.210000 seconds
   cpu MFLOPS      = 0.000000
 
 Problem size: 26460
@@ -113,30 +113,30 @@ AMS Setup:
   cpu clock time  = 0.100000 seconds
   cpu MFLOPS      = 0.000000
 
-<C*b,b>: 8.520115e-02
+<C*b,b>: 8.373120e-02
 
 
 Iters       ||r||_C     conv.rate  ||r||_C/||b||_C
 -----    ------------    ---------  ------------ 
-    1    2.578311e-02    0.088331    8.833088e-02
-    2    1.130413e-03    0.043843    3.872703e-03
-    3    8.998649e-05    0.079605    3.082865e-04
-    4    2.108336e-05    0.234295    7.222990e-05
-    5    3.535402e-06    0.167687    1.211201e-05
-    6    7.245863e-07    0.204952    2.482375e-06
-    7    1.246812e-07    0.172072    4.271479e-07
+    1    2.820328e-02    0.097467    9.746663e-02
+    2    1.245772e-03    0.044171    4.305215e-03
+    3    9.147965e-05    0.073432    3.161410e-04
+    4    2.203937e-05    0.240921    7.616500e-05
+    5    4.101633e-06    0.186105    1.417468e-05
+    6    7.804698e-07    0.190283    2.697195e-06
+    7    1.393405e-07    0.178534    4.815415e-07
 
 
 =============================================
 Solve phase times:
 =============================================
 AMS Solve:
-  wall clock time = 0.140000 seconds
+  wall clock time = 0.160000 seconds
   wall MFLOPS     = 0.000000
-  cpu clock time  = 0.150000 seconds
+  cpu clock time  = 0.160000 seconds
   cpu MFLOPS      = 0.000000
 
 
 Iterations = 7
-Final Relative Residual Norm = 4.27148e-07
+Final Relative Residual Norm = 4.81542e-07
 
diff --git a/src/test/TEST_examples/default.saved b/src/test/TEST_examples/default.saved
index c61ad53..e1cf87f 100644
--- a/src/test/TEST_examples/default.saved
+++ b/src/test/TEST_examples/default.saved
@@ -52,9 +52,9 @@ Final Relative Residual Norm = 9.21374e-08
 Setup phase times:
 =============================================
 PCG Setup:
-  wall clock time = 17.250000 seconds
+  wall clock time = 32.000000 seconds
   wall MFLOPS     = 0.000000
-  cpu clock time  = 14.520000 seconds
+  cpu clock time  = 25.090000 seconds
   cpu MFLOPS      = 0.000000
 
 <b,b>: 5.738541e-05
@@ -74,9 +74,9 @@ Iters       ||r||_2     conv.rate  ||r||_2/||b||_2
 Solve phase times:
 =============================================
 PCG Solve:
-  wall clock time = 0.050000 seconds
+  wall clock time = 8.160000 seconds
   wall MFLOPS     = 0.000000
-  cpu clock time  = 0.060000 seconds
+  cpu clock time  = 6.650000 seconds
   cpu MFLOPS      = 0.000000
 
 
@@ -287,9 +287,9 @@ Iters       ||r||_C     conv.rate  ||r||_C/||b||_C
 Setup phase times:
 =============================================
 PCG Setup:
-  wall clock time = 17.390000 seconds
+  wall clock time = 32.380000 seconds
   wall MFLOPS     = 0.000000
-  cpu clock time  = 13.680000 seconds
+  cpu clock time  = 25.270000 seconds
   cpu MFLOPS      = 0.000000
 
 <b,b>: 5.738541e-05
@@ -309,9 +309,9 @@ Iters       ||r||_2     conv.rate  ||r||_2/||b||_2
 Solve phase times:
 =============================================
 PCG Solve:
-  wall clock time = 0.070000 seconds
+  wall clock time = 7.260000 seconds
   wall MFLOPS     = 0.000000
-  cpu clock time  = 0.070000 seconds
+  cpu clock time  = 5.790000 seconds
   cpu MFLOPS      = 0.000000
 
 
@@ -368,7 +368,7 @@ Final Relative Residual Norm = 6.57147e-07
 * convergence tolerance     = 1.000000e-06
 *--------------------------------------------------
 AMG max levels   = 30
-AMG coarsen type = 6
+AMG coarsen type = 10
 AMG measure type = 0
 AMG threshold    = 2.500000e-01
 AMG numsweeps    = 1
@@ -389,35 +389,34 @@ AMG Schwarz relax weight = 1.000000e+00
 
 Iters       ||r||_2     conv.rate  ||r||_2/||b||_2
 -----    ------------   ---------  ------------ 
-    1    2.336684e-03    0.282758    2.827584e-01
-    2    1.512891e-04    0.064745    1.830725e-02
-    3    9.865125e-06    0.065207    1.193763e-03
-    4    4.401837e-07    0.044620    5.326593e-05
-    5    1.498060e-08    0.034033    1.812778e-06
-    6    5.485626e-10    0.036618    6.638068e-08
+    1    6.568705e-03    0.794868    7.948685e-01
+    2    2.229023e-03    0.339340    2.697305e-01
+    3    4.395103e-04    0.197176    5.318444e-02
+    4    6.154722e-05    0.140036    7.447730e-03
+    5    9.106693e-06    0.147963    1.101986e-03
+    6    1.448916e-06    0.159105    1.753310e-04
+    7    2.542814e-07    0.175498    3.077019e-05
+    8    3.559798e-08    0.139994    4.307655e-06
+    9    5.739829e-09    0.161240    6.945675e-07
 
 
 ***************************************************
 *                Solver Statistics                *
 *-------------------------------------------------*
-** HYPRE preconditioner setup time = 1.839185e-02
-** HYPRE solution time             = 1.583600e-02
-** HYPRE total time                = 3.422785e-02
-** HYPRE number of iterations      = 6
-** HYPRE final residual norm       = 5.485626e-10
+** HYPRE preconditioner setup time = 1.199627e-02
+** HYPRE solution time             = 1.896429e-02
+** HYPRE total time                = 3.096056e-02
+** HYPRE number of iterations      = 9
+** HYPRE final residual norm       = 5.739829e-09
 ***************************************************
 # Output file: default.out.11
-=================================================================
- 0  1089 x 545     1   4   2.500e-01 2.500e-01 7.500e-01 1.000e+00
- 1   545 x 157     1   4   7.143e-02 5.000e-01 2.857e-01 1.000e+00
- 2   157 x 57      1   6   2.767e-02 5.489e-01 2.630e-01 1.000e+00
- 3    57 x 19      0   5   1.892e-02 7.312e-01 0.000e+00 1.000e+00
- 4    19 x 6       0   4   2.675e-02 4.155e-01 0.000e+00 1.000e+00
+ 2   132 x 29      0   4   5.687e-03 5.328e-01 0.000e+00 1.000e+00
+ 3    29 x 4       1   4   6.881e-03 4.675e-01 7.079e-02 1.000e+00
 
 
-     Complexity:    grid = 1.719927
-                operator = 2.361942
-                memory = 3.194052
+     Complexity:    grid = 1.651974
+                operator = 2.383776
+                memory = 3.295125
 
 
 
@@ -430,11 +429,11 @@ BoomerAMG SOLVER PARAMETERS:
 
   Relaxation Parameters:
    Visiting Grid:                     down   up  coarse
-            Number of sweeps:            1    1     1 
-   Type 0=Jac, 3=hGS, 6=hSGS, 9=GE:      6    6     9 
+            Number of sweeps:            2    2     1 
+   Type 0=Jac, 3=hGS, 6=hSGS, 9=GE:     13   14     9 
    Point types, partial sweeps (1=C, -1=F):
-                  Pre-CG relaxation (down):   1  -1
-                   Post-CG relaxation (up):  -1   1
+                  Pre-CG relaxation (down):   0   0
+                   Post-CG relaxation (up):   0   0
                              Coarsest grid:   0
 
 
@@ -446,58 +445,62 @@ No constraints
 
 
 Initial Max. Residual   2.03558352288229e+00
-Iteration 1 	bsize 10 	maxres   6.61514157224843e-01
-Iteration 2 	bsize 10 	maxres   2.07896889117126e-01
-Iteration 3 	bsize 10 	maxres   8.94032219327286e-02
-Iteration 4 	bsize 10 	maxres   5.57350171829443e-02
-Iteration 5 	bsize 10 	maxres   4.52510317338198e-02
-Iteration 6 	bsize 10 	maxres   1.59857204735523e-02
-Iteration 7 	bsize 9 	maxres   3.01005214052435e-03
-Iteration 8 	bsize 8 	maxres   9.36952182472202e-04
-Iteration 9 	bsize 7 	maxres   2.77002984387563e-04
-Iteration 10 	bsize 5 	maxres   1.31048969636475e-04
-Iteration 11 	bsize 4 	maxres   5.87732297903442e-05
-Iteration 12 	bsize 3 	maxres   2.88066402410243e-05
-Iteration 13 	bsize 3 	maxres   2.04608190556959e-05
-Iteration 14 	bsize 2 	maxres   1.41674564734296e-05
-Iteration 15 	bsize 2 	maxres   9.49344900640354e-06
-Iteration 16 	bsize 2 	maxres   6.78846844902271e-06
-Iteration 17 	bsize 2 	maxres   3.98329316782246e-06
-Iteration 18 	bsize 2 	maxres   2.18494258745973e-06
-Iteration 19 	bsize 1 	maxres   1.23557151334132e-06
-Iteration 20 	bsize 1 	maxres   4.69008547559107e-07
-Iteration 21 	bsize 1 	maxres   3.62403218315907e-07
-Iteration 22 	bsize 1 	maxres   1.05430880785163e-07
-
-Eigenvalue lambda   1.70632948198618e-02
-Eigenvalue lambda   4.25854480421274e-02
-Eigenvalue lambda   4.25854480421300e-02
-Eigenvalue lambda   6.81076012643962e-02
-Eigenvalue lambda   8.48803610642934e-02
-Eigenvalue lambda   8.48803610642950e-02
-Eigenvalue lambda   1.10402514286555e-01
-Eigenvalue lambda   1.10402514286562e-01
-Eigenvalue lambda   1.43587188601058e-01
-Eigenvalue lambda   1.43587188601254e-01
-Residual   7.10733430591404e-09
-Residual   5.25724629473367e-08
-Residual   6.73692024719225e-09
-Residual   2.56416502787902e-08
-Residual   8.94205529543480e-08
-Residual   2.34185244900239e-08
-Residual   8.66934458230941e-08
-Residual   9.53360633931321e-08
-Residual   9.43660840688180e-08
-Residual   1.05430880785163e-07
-
-22 iterations
+Iteration 1 	bsize 10 	maxres   6.81010023232597e-01
+Iteration 2 	bsize 10 	maxres   2.16106638420483e-01
+Iteration 3 	bsize 10 	maxres   1.33255378957699e-01
+Iteration 4 	bsize 10 	maxres   6.63135103881281e-02
+Iteration 5 	bsize 10 	maxres   5.98898745939707e-02
+Iteration 6 	bsize 10 	maxres   3.24851727613947e-02
+Iteration 7 	bsize 10 	maxres   1.24818484659858e-02
+Iteration 8 	bsize 9 	maxres   3.28135782117784e-03
+Iteration 9 	bsize 9 	maxres   1.37465135021034e-03
+Iteration 10 	bsize 7 	maxres   5.53538771528421e-04
+Iteration 11 	bsize 6 	maxres   2.39420582335701e-04
+Iteration 12 	bsize 4 	maxres   1.65909557562297e-04
+Iteration 13 	bsize 4 	maxres   9.99715384081695e-05
+Iteration 14 	bsize 3 	maxres   5.93603896817596e-05
+Iteration 15 	bsize 2 	maxres   2.88451774920854e-05
+Iteration 16 	bsize 2 	maxres   1.03761599946811e-05
+Iteration 17 	bsize 2 	maxres   5.80576550088874e-06
+Iteration 18 	bsize 2 	maxres   4.76778988364290e-06
+Iteration 19 	bsize 2 	maxres   4.89371670567952e-06
+Iteration 20 	bsize 2 	maxres   3.10957559901931e-06
+Iteration 21 	bsize 2 	maxres   1.85952798560890e-06
+Iteration 22 	bsize 2 	maxres   9.96634709813028e-07
+Iteration 23 	bsize 2 	maxres   6.38856516889256e-07
+Iteration 24 	bsize 2 	maxres   3.57575647705605e-07
+Iteration 25 	bsize 1 	maxres   2.50720421537878e-07
+Iteration 26 	bsize 1 	maxres   1.33984380971654e-07
+
+Eigenvalue lambda   1.70632948198619e-02
+Eigenvalue lambda   4.25854480421056e-02
+Eigenvalue lambda   4.25854480421299e-02
+Eigenvalue lambda   6.81076012644266e-02
+Eigenvalue lambda   8.48803610641320e-02
+Eigenvalue lambda   8.48803610649201e-02
+Eigenvalue lambda   1.10402514286512e-01
+Eigenvalue lambda   1.10402514291700e-01
+Eigenvalue lambda   1.43587188598359e-01
+Eigenvalue lambda   1.43587188847947e-01
+Residual   2.59873942049052e-08
+Residual   1.18843349666549e-08
+Residual   1.23872744529267e-08
+Residual   5.10375475086747e-08
+Residual   5.10377811863076e-08
+Residual   3.70656076366488e-08
+Residual   3.63780666486010e-08
+Residual   1.18640460683670e-07
+Residual   1.21370637119926e-07
+Residual   1.33984380971654e-07
+
+26 iterations
 =============================================
 Solve phase times:
 =============================================
 LOBPCG Solve:
   wall clock time = 0.090000 seconds
   wall MFLOPS     = 0.000000
-  cpu clock time  = 0.080000 seconds
+  cpu clock time  = 0.090000 seconds
   cpu MFLOPS      = 0.000000
 
 # Output file: default.out.12
@@ -532,14 +535,17 @@ AMG SOLUTION INFO:
                residual        factor       residual
                --------        ------       --------
     Initial    2.334657e-01                 1.000000e+00
-    Cycle  1   1.343247e-02    0.057535     5.753510e-02 
-    Cycle  2   7.594482e-04    0.056538     3.252932e-03 
-    Cycle  3   4.273465e-05    0.056271     1.830446e-04 
-    Cycle  4   2.445691e-06    0.057230     1.047559e-05 
-    Cycle  5   1.471809e-07    0.060180     6.304178e-07 
+    Cycle  1   3.447912e-02    0.147684     1.476839e-01 
+    Cycle  2   5.073235e-03    0.147139     2.173011e-02 
+    Cycle  3   7.530408e-04    0.148434     3.225487e-03 
+    Cycle  4   1.131421e-04    0.150247     4.846199e-04 
+    Cycle  5   1.713572e-05    0.151453     7.339717e-05 
+    Cycle  6   2.620347e-06    0.152917     1.122369e-05 
+    Cycle  7   4.055800e-07    0.154781     1.737214e-06 
+    Cycle  8   6.362639e-08    0.156878     2.725299e-07 
 
 
- Average Convergence Factor = 0.057534
+ Average Convergence Factor = 0.151156
 
      Complexity:    grid = 1.509834
                 operator = 1.841964
@@ -548,8 +554,8 @@ AMG SOLUTION INFO:
 
 
 
-Iterations = 5
-Final Relative Residual Norm = 6.30418e-07
+Iterations = 8
+Final Relative Residual Norm = 2.7253e-07
 
 # Output file: default.out.14
 
@@ -559,14 +565,17 @@ AMG SOLUTION INFO:
                residual        factor       residual
                --------        ------       --------
     Initial    2.334657e-01                 1.000000e+00
-    Cycle  1   1.343247e-02    0.057535     5.753510e-02 
-    Cycle  2   7.594482e-04    0.056538     3.252932e-03 
-    Cycle  3   4.273465e-05    0.056271     1.830446e-04 
-    Cycle  4   2.445691e-06    0.057230     1.047559e-05 
-    Cycle  5   1.471809e-07    0.060180     6.304178e-07 
+    Cycle  1   3.447912e-02    0.147684     1.476839e-01 
+    Cycle  2   5.073235e-03    0.147139     2.173011e-02 
+    Cycle  3   7.530408e-04    0.148434     3.225487e-03 
+    Cycle  4   1.131421e-04    0.150247     4.846199e-04 
+    Cycle  5   1.713572e-05    0.151453     7.339717e-05 
+    Cycle  6   2.620347e-06    0.152917     1.122369e-05 
+    Cycle  7   4.055800e-07    0.154781     1.737214e-06 
+    Cycle  8   6.362639e-08    0.156878     2.725299e-07 
 
 
- Average Convergence Factor = 0.057534
+ Average Convergence Factor = 0.151156
 
      Complexity:    grid = 1.509834
                 operator = 1.841964
@@ -575,17 +584,17 @@ AMG SOLUTION INFO:
 
 
 
-Iterations = 5
-Final Relative Residual Norm = 6.30418e-07
+Iterations = 8
+Final Relative Residual Norm = 2.7253e-07
 
 # Output file: default.out.15
 =============================================
 SStruct phase times:
 =============================================
 SStruct Setup:
-  wall clock time = 0.250000 seconds
+  wall clock time = 0.220000 seconds
   wall MFLOPS     = 0.000000
-  cpu clock time  = 0.240000 seconds
+  cpu clock time  = 0.220000 seconds
   cpu MFLOPS      = 0.000000
 
 Problem size: 26460
@@ -594,35 +603,35 @@ Problem size: 26460
 Setup phase times:
 =============================================
 AMS Setup:
-  wall clock time = 0.110000 seconds
+  wall clock time = 0.090000 seconds
   wall MFLOPS     = 0.000000
-  cpu clock time  = 0.110000 seconds
+  cpu clock time  = 0.100000 seconds
   cpu MFLOPS      = 0.000000
 
-<C*b,b>: 8.520115e-02
+<C*b,b>: 8.373120e-02
 
 
 Iters       ||r||_C     conv.rate  ||r||_C/||b||_C
 -----    ------------    ---------  ------------ 
-    1    2.578311e-02    0.088331    8.833088e-02
-    2    1.130413e-03    0.043843    3.872703e-03
-    3    8.998649e-05    0.079605    3.082865e-04
-    4    2.108336e-05    0.234295    7.222990e-05
-    5    3.535402e-06    0.167687    1.211201e-05
-    6    7.245863e-07    0.204952    2.482375e-06
-    7    1.246812e-07    0.172072    4.271479e-07
+    1    2.820328e-02    0.097467    9.746663e-02
+    2    1.245772e-03    0.044171    4.305215e-03
+    3    9.147965e-05    0.073432    3.161410e-04
+    4    2.203937e-05    0.240921    7.616500e-05
+    5    4.101633e-06    0.186105    1.417468e-05
+    6    7.804698e-07    0.190283    2.697195e-06
+    7    1.393405e-07    0.178534    4.815415e-07
 
 
 =============================================
 Solve phase times:
 =============================================
 AMS Solve:
-  wall clock time = 0.160000 seconds
+  wall clock time = 0.150000 seconds
   wall MFLOPS     = 0.000000
-  cpu clock time  = 0.170000 seconds
+  cpu clock time  = 0.150000 seconds
   cpu MFLOPS      = 0.000000
 
 
 Iterations = 7
-Final Relative Residual Norm = 4.27148e-07
+Final Relative Residual Norm = 4.81542e-07
 
diff --git a/src/test/TEST_ij/agg_interp.jobs b/src/test/TEST_ij/agg_interp.jobs
index debd26d..096143c 100755
--- a/src/test/TEST_ij/agg_interp.jobs
+++ b/src/test/TEST_ij/agg_interp.jobs
@@ -12,7 +12,7 @@
 #EHEADER**********************************************************************
 
 #=============================================================================
-# new_ij: Test aggressive coarsening interpolation options
+# ij: Test aggressive coarsening interpolation options
 #     1: 2s-ei interpolation 1 level (Pmx = 4)
 #     2: 2s-std interpolation 1 level (Pmx = 4)
 #     3: 2s-ext interpolation 1 level (tr = 0.3)
@@ -28,42 +28,42 @@
 #     8: 2s-ei interpolation all levels for systems problem hybrid approach
 #=============================================================================
 
-mpirun -np 8 ./new_ij -rhsrand -n 30 29 31 -P 2 2 2 -hmis -agg_nl 1 -agg_interp 1 -agg_Pmx 4 -solver 1 -rlx 6 \
+mpirun -np 8 ./ij    -rhsrand -n 30 29 31 -P 2 2 2 -agg_nl 1 -agg_interp 1 -agg_Pmx 4 -solver 1 -rlx 6 \
  > agg_interp.out.1
 
-mpirun -np 8 ./new_ij -rhsrand -n 30 29 31 -P 2 2 2 -hmis -agg_nl 1 -agg_interp 2 -agg_Pmx 4 -solver 1 -rlx 6 \
+mpirun -np 8 ./ij    -rhsrand -n 30 29 31 -P 2 2 2 -agg_nl 1 -agg_interp 2 -agg_Pmx 4 -solver 1 -rlx 6 \
  >> agg_interp.out.2
 
-mpirun -np 8 ./new_ij -rhsrand -n 30 29 31 -P 2 2 2 -hmis -agg_nl 1 -agg_interp 3 -agg_tr 0.3 -solver 1 -rlx 6 \
+mpirun -np 8 ./ij    -rhsrand -n 30 29 31 -P 2 2 2 -agg_nl 1 -agg_interp 3 -agg_tr 0.3 -solver 1 -rlx 6 \
  >> agg_interp.out.3
 
-mpirun -np 8 ./new_ij -rhsrand -n 30 29 31 -P 2 2 2 -hmis -agg_nl 1 -agg_interp 4  -solver 1 -rlx 6 \
+mpirun -np 8 ./ij    -rhsrand -n 30 29 31 -P 2 2 2 -agg_nl 1 -agg_interp 4  -solver 1 -rlx 6 \
  >> agg_interp.out.4
 
-mpirun -np 8 ./new_ij -rhsrand -n 30 29 31 -P 2 2 2 -hmis -agg_nl 10 -agg_interp 1 -agg_Pmx 4 -solver 1 -rlx 6 \
+mpirun -np 8 ./ij    -rhsrand -n 30 29 31 -P 2 2 2 -agg_nl 10 -agg_interp 1 -agg_Pmx 4 -solver 1 -rlx 6 \
  > agg_interp.out.5
 
-mpirun -np 8 ./new_ij -rhsrand -n 30 29 31 -P 2 2 2 -hmis -agg_nl 10 -agg_interp 2 -agg_Pmx 4 -solver 1 -rlx 6 \
+mpirun -np 8 ./ij    -rhsrand -n 30 29 31 -P 2 2 2 -agg_nl 10 -agg_interp 2 -agg_Pmx 4 -solver 1 -rlx 6 \
  >> agg_interp.out.6
 
-mpirun -np 8 ./new_ij -rhsrand -n 30 29 31 -P 2 2 2 -hmis -agg_nl 10 -agg_interp 3 -agg_tr 0.3 -agg_P12_tr 0.2 -solver 1 -rlx 6 \
+mpirun -np 8 ./ij    -rhsrand -n 30 29 31 -P 2 2 2 -agg_nl 10 -agg_interp 3 -agg_tr 0.3 -agg_P12_tr 0.2 -solver 1 -rlx 6 \
  >> agg_interp.out.7
 
-mpirun -np 8 ./new_ij -rhsrand -n 30 29 31 -P 2 2 2 -hmis -agg_nl 10 -agg_interp 4 -solver 1 -rlx 6 \
+mpirun -np 8 ./ij    -rhsrand -n 30 29 31 -P 2 2 2 -agg_nl 10 -agg_interp 4 -solver 1 -rlx 6 \
  >> agg_interp.out.8
 
-mpirun -np 8 ./new_ij -rhsrand -n 30 29 31 -P 2 2 2 -hmis -agg_nl 1 -agg_interp 1 -agg_Pmx 4 -agg_P12_mx 3 -solver 1 -rlx 6 \
+mpirun -np 8 ./ij    -rhsrand -n 30 29 31 -P 2 2 2 -agg_nl 1 -agg_interp 1 -agg_Pmx 4 -agg_P12_mx 3 -solver 1 -rlx 6 \
  > agg_interp.out.9
 
-mpirun -np 8 ./new_ij -rhsrand -n 20 19 22 -P 2 2 2 -sysL 3 -nf 3 -hmis -agg_nl 10 -agg_interp 4 -solver 1 -rlx 6 \
+mpirun -np 8 ./ij    -rhsrand -n 20 19 22 -P 2 2 2 -sysL 3 -nf 3 -agg_nl 10 -agg_interp 4 -solver 1 -rlx 6 \
  >> agg_interp.out.10
 
-mpirun -np 8 ./new_ij -rhsrand -n 20 19 22 -P 2 2 2 -sysL 3 -nf 3 -hmis -agg_nl 10 -agg_interp 1 -agg_Pmx 4 -agg_P12_mx 4 -solver 1 -rlx 6 \
+mpirun -np 8 ./ij    -rhsrand -n 20 19 22 -P 2 2 2 -sysL 3 -nf 3 -agg_nl 10 -agg_interp 1 -agg_Pmx 4 -agg_P12_mx 4 -solver 1 -rlx 6 \
  >> agg_interp.out.11
 
-mpirun -np 8 ./new_ij -rhsrand -n 20 19 22 -P 2 2 2 -sysL 3 -nf 3 -hmis -agg_nl 10 -agg_interp 4 -nodal 1 -solver 1 -rlx 6 \
+mpirun -np 8 ./ij    -rhsrand -n 20 19 22 -P 2 2 2 -sysL 3 -nf 3 -agg_nl 10 -agg_interp 4 -nodal 1 -solver 1 -rlx 6 \
  >> agg_interp.out.12
 
-mpirun -np 8 ./new_ij -rhsrand -n 20 19 22 -P 2 2 2 -sysL 3 -nf 3 -hmis -agg_nl 10 -agg_interp 1 -agg_Pmx 4 -agg_P12_mx 4 -nodal 1 -solver 1 -rlx 6 \
+mpirun -np 8 ./ij    -rhsrand -n 20 19 22 -P 2 2 2 -sysL 3 -nf 3 -agg_nl 10 -agg_interp 1 -agg_Pmx 4 -agg_P12_mx 4 -nodal 1 -solver 1 -rlx 6 \
  >> agg_interp.out.13
 
diff --git a/src/test/TEST_ij/agg_interp.saved b/src/test/TEST_ij/agg_interp.saved
index ff3982c..5c5f985 100644
--- a/src/test/TEST_ij/agg_interp.saved
+++ b/src/test/TEST_ij/agg_interp.saved
@@ -1,78 +1,78 @@
 # Output file: agg_interp.out.1
 
 
-Iterations = 10
-Final Relative Residual Norm = 2.324566e-09
+Iterations = 9
+Final Relative Residual Norm = 4.609095e-09
 
 # Output file: agg_interp.out.2
 
 
 Iterations = 9
-Final Relative Residual Norm = 8.317304e-09
+Final Relative Residual Norm = 3.785900e-09
 
 # Output file: agg_interp.out.3
 
 
-Iterations = 11
-Final Relative Residual Norm = 4.394947e-09
+Iterations = 10
+Final Relative Residual Norm = 7.511453e-09
 
 # Output file: agg_interp.out.4
 
 
-Iterations = 13
-Final Relative Residual Norm = 4.604995e-09
+Iterations = 12
+Final Relative Residual Norm = 3.507939e-09
 
 # Output file: agg_interp.out.5
 
 
 Iterations = 10
-Final Relative Residual Norm = 9.551604e-09
+Final Relative Residual Norm = 7.390297e-09
 
 # Output file: agg_interp.out.6
 
 
 Iterations = 10
-Final Relative Residual Norm = 7.249545e-09
+Final Relative Residual Norm = 5.579282e-09
 
 # Output file: agg_interp.out.7
 
 
 Iterations = 12
-Final Relative Residual Norm = 7.742892e-09
+Final Relative Residual Norm = 5.781132e-09
 
 # Output file: agg_interp.out.8
 
 
 Iterations = 15
-Final Relative Residual Norm = 9.503769e-09
+Final Relative Residual Norm = 5.637036e-09
 
 # Output file: agg_interp.out.9
 
 
-Iterations = 11
-Final Relative Residual Norm = 2.452371e-09
+Iterations = 10
+Final Relative Residual Norm = 4.335635e-09
 
 # Output file: agg_interp.out.10
 
 
-Iterations = 23
-Final Relative Residual Norm = 6.103183e-09
+Iterations = 22
+Final Relative Residual Norm = 8.737365e-09
 
 # Output file: agg_interp.out.11
 
 
-Iterations = 18
-Final Relative Residual Norm = 4.471484e-09
+Iterations = 17
+Final Relative Residual Norm = 9.807905e-09
 
 # Output file: agg_interp.out.12
 
 
 Iterations = 20
-Final Relative Residual Norm = 5.905785e-09
+Final Relative Residual Norm = 5.870410e-09
 
 # Output file: agg_interp.out.13
 
 
 Iterations = 16
-Final Relative Residual Norm = 5.103180e-09
+Final Relative Residual Norm = 5.087936e-09
 
diff --git a/src/test/TEST_ij/coarsening.jobs b/src/test/TEST_ij/coarsening.jobs
index 41b11b3..a743340 100755
--- a/src/test/TEST_ij/coarsening.jobs
+++ b/src/test/TEST_ij/coarsening.jobs
@@ -21,40 +21,54 @@
 # ij: Run default case with different coarsenings, hybrid GS,
 #    1: Cleary_LJP
 #    2: parallel Ruge
-#    3: Ruge 3rd pass
+#    3: Ruge 3rd pass, keep c points on boundary
 #    4: Falgout
-# new_ij: Run default case with low complexity coarsenings, hybrid GS,
+# ij: Run default case with low complexity coarsenings, hybrid GS,
 #    5: PMIS
 #    6: HMIS
 #    7: Falgout 2 levels aggressive coarsening
 #    8: HMIS 1 level aggressive coarsening
+#    9: Ruge 1 pass with global measures
+#   10: Ruge 1 pass coarsening
+#   11: CGC coarsening
+#   12: Ruge 3 coarsening
+#   13: CLJP1 fixed random coarsening
+#   14: PMIS1 fixed random coarsening
 #=============================================================================
 
-mpirun -np 4 ./ij -rhsrand -n 15 15 10 -P 2 2 1 -cljp -27pt \
+mpirun -np 4 ./ij -rhsrand -n 15 15 10 -P 2 2 1 -interptype 0 -Pmx 0 -cljp -27pt \
  > coarsening.out.0
 
-mpirun -np 4 ./ij -rhsrand -n 15 15 10 -P 2 2 1 -ruge -27pt \
+mpirun -np 4 ./ij -rhsrand -n 15 15 10 -P 2 2 1 -interptype 0 -Pmx 0 -ruge -27pt \
  > coarsening.out.1
 
-mpirun -np 4 ./ij -rhsrand -n 15 15 10 -P 2 2 1 -ruge3c -gm -27pt \
+mpirun -np 4 ./ij -rhsrand -n 15 15 10 -P 2 2 1 -interptype 0 -Pmx 0 -ruge3c -gm -27pt \
  > coarsening.out.2
 
-mpirun -np 4 ./ij -rhsrand -n 15 15 10 -P 2 2 1 -falgout -27pt \
+mpirun -np 4 ./ij -rhsrand -n 15 15 10 -P 2 2 1 -interptype 0 -Pmx 0 -falgout -27pt \
  > coarsening.out.3
 
-mpirun -np 4 ./new_ij -rhsrand -n 15 15 10 -P 2 2 1 -pmis -27pt \
+mpirun -np 4 ./ij -rhsrand -n 15 15 10 -P 2 2 1 -pmis -27pt \
  > coarsening.out.4
 
-mpirun -np 4 ./new_ij -rhsrand -n 15 15 10 -P 2 2 1 -hmis -27pt \
+mpirun -np 4 ./ij -rhsrand -n 15 15 10 -P 2 2 1 -27pt \
  > coarsening.out.5
 
-mpirun -np 4 ./new_ij -rhsrand -n 60 60 30 -P 2 2 1 -falgout -agg_nl 2 -27pt -rlx 6 -solver 1 \
+mpirun -np 4 ./ij -rhsrand -n 60 60 30 -P 2 2 1 -falgout -interptype 0 -Pmx 0 -agg_nl 2 -27pt -rlx 6 -solver 1 \
  > coarsening.out.6
 
-mpirun -np 4 ./new_ij -rhsrand -n 60 60 30 -P 2 2 1 -hmis -agg_nl 1 -27pt -rlx 6 -solver 1 \
+mpirun -np 4 ./ij -rhsrand -n 60 60 30 -P 2 2 1 -agg_nl 1 -27pt -rlx 6 -solver 1 \
  > coarsening.out.7
 
-mpirun -np 2 ./new_ij -hmis -gm > coarsening.out.8
+mpirun -np 2 ./ij -ruge1p -gm > coarsening.out.8
 
-mpirun -np 8 ./new_ij -cgc > coarsening.out.9
+mpirun -np 8 ./ij -P 2 2 2 -ruge1p  > coarsening.out.9
+
+mpirun -np 8 ./ij -P 2 2 2 -cgc -interptype 0 -Pmx 0 > coarsening.out.10
+
+mpirun -np 8 ./ij -P 2 2 2 -ruge3  -interptype 0 -Pmx 0 > coarsening.out.11
+
+mpirun -np 8 ./ij -P 2 2 2 -cljp1  -interptype 0 -Pmx 0 > coarsening.out.12
+
+mpirun -np 8 ./ij -P 2 2 2 -pmis1 > coarsening.out.13
 
diff --git a/src/test/TEST_ij/coarsening.saved b/src/test/TEST_ij/coarsening.saved
index 34681f6..3537da5 100644
--- a/src/test/TEST_ij/coarsening.saved
+++ b/src/test/TEST_ij/coarsening.saved
@@ -1,58 +1,74 @@
 # Output file: coarsening.out.0
- Average Convergence Factor = 0.037296
+ Average Convergence Factor = 0.094028
 
      Complexity:    grid = 1.220000
                 operator = 1.408310
                    cycle = 2.816445
 
 # Output file: coarsening.out.1
- Average Convergence Factor = 0.056033
+ Average Convergence Factor = 0.124105
 
      Complexity:    grid = 1.190222
                 operator = 1.269219
                    cycle = 2.538129
 
 # Output file: coarsening.out.2
- Average Convergence Factor = 0.053501
+ Average Convergence Factor = 0.121893
 
      Complexity:    grid = 1.236444
                 operator = 1.378931
                    cycle = 2.756625
 
 # Output file: coarsening.out.3
- Average Convergence Factor = 0.044811
+ Average Convergence Factor = 0.114604
 
      Complexity:    grid = 1.204889
                 operator = 1.315634
                    cycle = 2.631094
 
 # Output file: coarsening.out.4
- Average Convergence Factor = 0.200231
+ Average Convergence Factor = 0.126924
 
-     Complexity:    grid = 1.101778
-                operator = 1.078942
-                   cycle = 2.157576
+     Complexity:    grid = 1.104889
+                operator = 1.161632
+                   cycle = 2.322568
 
 # Output file: coarsening.out.5
- Average Convergence Factor = 0.109219
+ Average Convergence Factor = 0.115880
 
-     Complexity:    grid = 1.126667
-                operator = 1.110426
-                   cycle = 2.219907
+     Complexity:    grid = 1.130222
+                operator = 1.225044
+                   cycle = 2.449915
 
 # Output file: coarsening.out.6
 Iterations = 12
-Final Relative Residual Norm = 4.082574e-09
+Final Relative Residual Norm = 3.821883e-09
 
 # Output file: coarsening.out.7
-Iterations = 12
-Final Relative Residual Norm = 3.409513e-09
+Iterations = 11
+Final Relative Residual Norm = 7.350494e-09
 
 # Output file: coarsening.out.8
-BoomerAMG Iterations = 9
-Final Relative Residual Norm = 7.825251e-09
+BoomerAMG Iterations = 13
+Final Relative Residual Norm = 3.043813e-09
 
 # Output file: coarsening.out.9
-BoomerAMG Iterations = 7
-Final Relative Residual Norm = 1.174752e-09
+BoomerAMG Iterations = 15
+Final Relative Residual Norm = 3.616706e-09
+
+# Output file: coarsening.out.10
+BoomerAMG Iterations = 15
+Final Relative Residual Norm = 4.933005e-09
+
+# Output file: coarsening.out.11
+BoomerAMG Iterations = 13
+Final Relative Residual Norm = 5.238460e-09
+
+# Output file: coarsening.out.12
+BoomerAMG Iterations = 13
+Final Relative Residual Norm = 2.784139e-09
+
+# Output file: coarsening.out.13
+BoomerAMG Iterations = 14
+Final Relative Residual Norm = 3.301634e-09
 
diff --git a/src/test/TEST_ij/coarsening.sh b/src/test/TEST_ij/coarsening.sh
index f99ab20..ea2b50d 100755
--- a/src/test/TEST_ij/coarsening.sh
+++ b/src/test/TEST_ij/coarsening.sh
@@ -37,6 +37,10 @@ FILES="\
  ${TNAME}.out.7\
  ${TNAME}.out.8\
  ${TNAME}.out.9\
+ ${TNAME}.out.10\
+ ${TNAME}.out.11\
+ ${TNAME}.out.12\
+ ${TNAME}.out.13\
 "
 
 for i in $FILES
diff --git a/src/test/TEST_ij/default.jobs b/src/test/TEST_ij/default.jobs
index d568241..e523348 100755
--- a/src/test/TEST_ij/default.jobs
+++ b/src/test/TEST_ij/default.jobs
@@ -18,12 +18,12 @@
 
 
 #=============================================================================
-# ij: Run default case, weigthed Jacobi, BoomerAMG
+# ij: Run default case (first old, then new), CF Jacobi, BoomerAMG
 #=============================================================================
 
-mpirun -np 1 ./ij -cljp -rlx 0 -xisone > default.out.0
+mpirun -np 1 ./ij -pmis -Pmx 0 -rlx 0 -xisone > default.out.0
 
-mpirun -np 2 ./ij -P 1 1 2 -cljp1 -rlx 0 -xisone > default.out.1
+mpirun -np 2 ./ij -P 1 1 2 -pmis1 -Pmx 0 -rlx 0 -xisone > default.out.1
 
-mpirun -np 3 ./ij -P 1 1 3 -cljp1 -rlx 0 -xisone > default.out.2
+mpirun -np 3 ./ij -P 1 1 3 -pmis1 -Pmx 0 -rlx 0 -xisone > default.out.2
 
diff --git a/src/test/TEST_ij/default.saved b/src/test/TEST_ij/default.saved
index 3af177c..df37a26 100644
--- a/src/test/TEST_ij/default.saved
+++ b/src/test/TEST_ij/default.saved
@@ -1,21 +1,21 @@
 # Output file: default.out.0
- Average Convergence Factor = 0.044862
+ Average Convergence Factor = 0.678738
 
-     Complexity:    grid = 2.237000
-                operator = 7.017969
-                   cycle = 14.033437
+     Complexity:    grid = 1.407000
+                operator = 3.252344
+                   cycle = 6.499062
 
 # Output file: default.out.1
- Average Convergence Factor = 0.044862
+ Average Convergence Factor = 0.678738
 
-     Complexity:    grid = 2.237000
-                operator = 7.017969
-                   cycle = 14.033437
+     Complexity:    grid = 1.407000
+                operator = 3.252344
+                   cycle = 6.499062
 
 # Output file: default.out.2
- Average Convergence Factor = 0.044862
+ Average Convergence Factor = 0.678738
 
-     Complexity:    grid = 2.237000
-                operator = 7.017969
-                   cycle = 14.033437
+     Complexity:    grid = 1.407000
+                operator = 3.252344
+                   cycle = 6.499062
 
diff --git a/src/test/TEST_ij/elast.jobs b/src/test/TEST_ij/elast.jobs
index f1d834e..bb6aaa4 100755
--- a/src/test/TEST_ij/elast.jobs
+++ b/src/test/TEST_ij/elast.jobs
@@ -18,22 +18,32 @@
 
 
 #=============================================================================
-# new_ij: Run 2D elasticity problem with different interpolation operators
-#    1: unknown approach
-#    2: hybrid approach
-#    3: GM variant 1
-#    4: GM variant 2 Qmx=2
-#    5: GM variant 2 Qtr=0.01
-#    5: LN approach
-#    6: LN approach with 3 RBMs (2 more than needed) 
+# ij: Run 2D elasticity problem with different interpolation operators
+#    1: unknown approach (old default)
+#    2: hybrid approach (old default)
+#    3: GM variant 1 (old default)
+#    4: GM variant 2 Qmx=2 (old default)
+#    5: GM variant 2 Qtr=0.01 (old default)
+#    5: LN approach (old default)
+#    6: LN approach with 3 RBMs (2 more than needed) (old default)
+#    7: unknown approach with default settings
+#    8: hybrid approach with default settings
+#    9: GM variant 2 Qmx=2 with default settings
+#   10: classical block interpolation for nodal systems AMG
+#   11: classical block interpolation for nodal systems AMG with diagonal blocks
 #=============================================================================
 
-mpirun -np 2 ./new_ij -fromfile A -nf 2 -rlx 6 -solver 1 > elast.out.0
-mpirun -np 2 ./new_ij -fromfile A -nf 2 -rlx 6 -solver 1 -nodal 4 > elast.out.1
-mpirun -np 2 ./new_ij -fromfile A -rbm 1 rbm -nf 2 -interpvecvar 1 -nodal 1 -rlx 6 -solver 1 > elast.out.2
-mpirun -np 2 ./new_ij -fromfile A -rbm 1 rbm -nf 2 -interpvecvar 1 -nodal 4 -rlx 6 -solver 1 -Qmx 2 > elast.out.3
-mpirun -np 2 ./new_ij -fromfile A -rbm 1 rbm -nf 2 -interpvecvar 2 -nodal 3 -rlx 6 -solver 1 -Qtr 0.01 > elast.out.4
-mpirun -np 2 ./new_ij -fromfile A -rbm 1 rbm -nf 2 -interpvecvar 3 -nodal 6 -rlx 6 -solver 1 > elast.out.5
-mpirun -np 2 ./new_ij -fromfile A -rbm 3 rbm -nf 2 -interpvecvar 3 -nodal 6 -rlx 6 -solver 1 > elast.out.6
+mpirun -np 2 ./ij -fromfile A -nf 2 -rlx 6 -solver 1 -falgout -interptype 0 -Pmx 0 > elast.out.0
+mpirun -np 2 ./ij -fromfile A -nf 2 -rlx 6 -solver 1 -nodal 4 -falgout -interptype 0 -Pmx 0 > elast.out.1
+mpirun -np 2 ./ij -fromfile A -rbm 1 rbm -nf 2 -interpvecvar 1 -nodal 1 -rlx 6 -solver 1 -falgout -interptype 0 -Pmx 0 > elast.out.2
+mpirun -np 2 ./ij -fromfile A -rbm 1 rbm -nf 2 -interpvecvar 1 -nodal 4 -rlx 6 -solver 1 -falgout -interptype 0 -Pmx 0 -Qmx 2 > elast.out.3
+mpirun -np 2 ./ij -fromfile A -rbm 1 rbm -nf 2 -interpvecvar 2 -nodal 3 -rlx 6 -solver 1 -falgout -interptype 0 -Pmx 0 -Qtr 0.01 > elast.out.4
+mpirun -np 2 ./ij -fromfile A -rbm 1 rbm -nf 2 -interpvecvar 3 -nodal 6 -rlx 6 -solver 1 -falgout -interptype 0 -Pmx 0 > elast.out.5
+mpirun -np 2 ./ij -fromfile A -rbm 3 rbm -nf 2 -interpvecvar 3 -nodal 6 -rlx 6 -solver 1 -falgout -interptype 0 -Pmx 0 > elast.out.6
+mpirun -np 2 ./ij -fromfile A -nf 2 -rlx 6 -solver 1  > elast.out.7
+mpirun -np 2 ./ij -fromfile A -nf 2 -rlx 6 -solver 1 -nodal 4 > elast.out.8
+mpirun -np 2 ./ij -fromfile A -rbm 1 rbm -nf 2 -interpvecvar 2 -Qmx 2 -nodal 1 -rlx 6 -solver 1 > elast.out.9
+mpirun -np 2 ./ij -fromfile A -nf 2 -rlx 26 -interptype 10 -solver 1 > elast.out.9
+mpirun -np 2 ./ij -fromfile A -nf 2 -rlx 26 -interptype 11 -solver 1 > elast.out.9
 
 
diff --git a/src/test/TEST_ij/elast.saved b/src/test/TEST_ij/elast.saved
index 92056ec..fbe78e4 100644
--- a/src/test/TEST_ij/elast.saved
+++ b/src/test/TEST_ij/elast.saved
@@ -1,49 +1,70 @@
 # Output file: elast.out.0
-    9    1.775702e-03    0.275666    5.713229e-05
-   10    1.974196e-04    0.111178    6.351872e-06
-   11    2.332228e-05    0.118136    7.503822e-07
-   12    3.284387e-06    0.140826    1.056734e-07
-   13    3.127485e-07    0.095223    1.006252e-08
-   14    5.793189e-08    0.185235    1.863928e-09
+   10    1.020793e-03    0.147854    3.284349e-05
+   11    1.829583e-04    0.179231    5.886586e-06
+   12    2.804552e-05    0.153289    9.023500e-07
+   13    4.893103e-06    0.174470    1.574330e-07
+   14    8.444638e-07    0.172582    2.717018e-08
+   15    9.421889e-08    0.111572    3.031443e-09
 # Output file: elast.out.1
-   15    2.689279e-04    0.391423    8.652612e-06
-   16    4.472515e-05    0.166309    1.439008e-06
-   17    1.092863e-05    0.244351    3.516229e-07
-   18    3.320813e-06    0.303864    1.068454e-07
-   19    5.895465e-07    0.177531    1.896835e-08
-   20    1.068413e-07    0.181226    3.437561e-09
+   14    6.166494e-04    0.164100    1.984037e-05
+   15    1.391252e-04    0.225615    4.476281e-06
+   16    2.117573e-05    0.152206    6.813178e-07
+   17    2.872939e-06    0.135671    9.243531e-08
+   18    6.923073e-07    0.240975    2.227462e-08
+   19    1.641146e-07    0.237055    5.280300e-09
 # Output file: elast.out.2
-   13    8.039888e-04    0.402634    2.586791e-05
-   14    1.116905e-04    0.138920    3.593582e-06
-   15    2.112141e-05    0.189107    6.795703e-07
-   16    3.544248e-06    0.167804    1.140343e-07
-   17    6.603106e-07    0.186305    2.124515e-08
-   18    9.733195e-08    0.147403    3.131604e-09
+   12    8.670556e-04    0.177184    2.789706e-05
+   13    3.153597e-04    0.363713    1.014653e-05
+   14    5.627161e-05    0.178436    1.810509e-06
+   15    5.278098e-06    0.093797    1.698200e-07
+   16    8.783279e-07    0.166410    2.825974e-08
+   17    1.556180e-07    0.177175    5.006927e-09
 # Output file: elast.out.3
-   13    4.587599e-04    0.583506    1.476036e-05
-   14    1.016257e-04    0.221523    3.269752e-06
-   15    2.225654e-05    0.219005    7.160926e-07
-   16    3.152272e-06    0.141633    1.014227e-07
-   17    6.036038e-07    0.191482    1.942064e-08
-   18    1.251429e-07    0.207326    4.026407e-09
+   13    2.845966e-04    0.329643    9.156746e-06
+   14    5.961917e-05    0.209487    1.918215e-06
+   15    8.522551e-06    0.142950    2.742086e-07
+   16    1.725429e-06    0.202455    5.551478e-08
+   17    5.059175e-07    0.293213    1.627763e-08
+   18    5.680426e-08    0.112280    1.827647e-09
 # Output file: elast.out.4
-   13    6.645522e-04    0.241924    2.138162e-05
-   14    1.190138e-04    0.179089    3.829207e-06
-   15    4.180072e-05    0.351226    1.344916e-06
-   16    7.872870e-06    0.188343    2.533054e-07
-   17    2.163595e-06    0.274817    6.961253e-08
-   18    3.099808e-07    0.143271    9.973469e-09
+   13    7.466792e-04    0.200931    2.402401e-05
+   14    1.578734e-04    0.211434    5.079493e-06
+   15    4.801946e-05    0.304164    1.545001e-06
+   16    1.534974e-05    0.319657    4.938699e-07
+   17    2.491226e-06    0.162298    8.015387e-08
+   18    3.067633e-07    0.123137    9.869947e-09
 # Output file: elast.out.5
-   10    7.552044e-04    0.134393    2.429830e-05
-   11    2.672271e-04    0.353847    8.597892e-06
-   12    2.915202e-05    0.109091    9.379510e-07
-   13    8.077964e-06    0.277098    2.599042e-07
-   14    1.083827e-06    0.134171    3.487155e-08
-   15    1.257991e-07    0.116069    4.047520e-09
+    9    9.841674e-04    0.097482    3.166507e-05
+   10    2.375490e-04    0.241371    7.643014e-06
+   11    3.406966e-05    0.143422    1.096173e-06
+   12    5.837305e-06    0.171334    1.878122e-07
+   13    1.380887e-06    0.236562    4.442930e-08
+   14    2.272992e-07    0.164604    7.313232e-09
 # Output file: elast.out.6
-    9    2.300891e-03    0.305673    7.402996e-05
-   10    2.972382e-04    0.129184    9.563483e-06
-   11    7.855531e-05    0.264284    2.527476e-06
-   12    1.049780e-05    0.133636    3.377611e-07
-   13    1.211172e-06    0.115374    3.896883e-08
-   14    2.104243e-07    0.173736    6.770292e-09
+    9    6.193797e-04    0.172624    1.992822e-05
+   10    1.080160e-04    0.174394    3.475357e-06
+   11    1.217296e-05    0.112696    3.916587e-07
+   12    2.490164e-06    0.204565    8.011970e-08
+   13    4.278125e-07    0.171801    1.376464e-08
+   14    3.196541e-08    0.074718    1.028470e-09
+# Output file: elast.out.7
+   11    1.391109e-03    0.250496    4.475819e-05
+   12    1.718162e-04    0.123510    5.528094e-06
+   13    2.271768e-05    0.132221    7.309294e-07
+   14    3.733169e-06    0.164329    1.201127e-07
+   15    7.783590e-07    0.208498    2.504329e-08
+   16    5.833470e-08    0.074946    1.876888e-09
+# Output file: elast.out.8
+   20    2.046157e-04    0.413149    6.583403e-06
+   21    2.694826e-05    0.131702    8.670460e-07
+   22    2.020228e-05    0.749669    6.499976e-07
+   23    5.223484e-06    0.258559    1.680628e-07
+   24    1.683876e-06    0.322366    5.417781e-08
+   25    1.847586e-07    0.109722    5.944511e-09
+# Output file: elast.out.9
+   17    3.404660e-05    0.165262    1.095431e-06
+   18    1.402890e-05    0.412050    4.513724e-07
+   19    5.152788e-06    0.367298    1.657882e-07
+   20    9.630675e-07    0.186902    3.098619e-08
+   21    3.720854e-07    0.386354    1.197165e-08
+   22    8.377304e-08    0.225145    2.695353e-09
diff --git a/src/test/TEST_ij/elast.sh b/src/test/TEST_ij/elast.sh
index 0cbc23e..d8670fb 100755
--- a/src/test/TEST_ij/elast.sh
+++ b/src/test/TEST_ij/elast.sh
@@ -25,6 +25,9 @@ FILES="\
  ${TNAME}.out.4\
  ${TNAME}.out.5\
  ${TNAME}.out.6\
+ ${TNAME}.out.7\
+ ${TNAME}.out.8\
+ ${TNAME}.out.9\
 "
 
 for i in $FILES
diff --git a/src/test/TEST_ij/interp.jobs b/src/test/TEST_ij/interp.jobs
index 691811e..c794430 100755
--- a/src/test/TEST_ij/interp.jobs
+++ b/src/test/TEST_ij/interp.jobs
@@ -18,31 +18,35 @@
 
 
 #=============================================================================
-# new_ij: Run default case with different interpolation operators
+# ij: Run default case with different interpolation operators
 #    1: Extended interpolation
 #    2: FF interpolation
 #    3: standard interpolation
 #    4: Extended interpolation and truncation max 4 elmts per row
 #    5: FF interpolation and truncation max 4 elmts per row
 #    6: standard interpolation and truncation max 4 elmts per row
+#    7: Classical modified interpolation
 #=============================================================================
 
-mpirun -np 4 ./new_ij -rhsrand -n 15 15 10 -P 2 2 1 -pmis -interptype 6 \
+mpirun -np 4 ./ij -rhsrand -n 15 15 10 -P 2 2 1 -Pmx 0 \
  > interp.out.0
 
-mpirun -np 4 ./new_ij -rhsrand -n 15 15 10 -P 2 2 1 -pmis -interptype 7 \
+mpirun -np 4 ./ij -rhsrand -n 15 15 10 -P 2 2 1 -interptype 7 -Pmx 0\
  > interp.out.1
 
-mpirun -np 4 ./new_ij -rhsrand -n 15 15 10 -P 2 2 1 -pmis -interptype 8 \
+mpirun -np 4 ./ij -rhsrand -n 15 15 10 -P 2 2 1 -interptype 8 -Pmx 0 \
  > interp.out.2
 
-mpirun -np 4 ./new_ij -rhsrand -n 15 15 10 -P 2 2 1 -pmis -interptype 6 -Pmx 4 \
+mpirun -np 4 ./ij -rhsrand -n 15 15 10 -P 2 2 1 \
  > interp.out.3
 
-mpirun -np 4 ./new_ij -rhsrand -n 15 15 10 -P 2 2 1 -pmis -interptype 7 -Pmx 4 \
+mpirun -np 4 ./ij -rhsrand -n 15 15 10 -P 2 2 1 -interptype 7 \
  > interp.out.4
 
-mpirun -np 4 ./new_ij -rhsrand -n 15 15 10 -P 2 2 1 -pmis -interptype 8 -Pmx 4 \
+mpirun -np 4 ./ij -rhsrand -n 15 15 10 -P 2 2 1 -interptype 8 \
  > interp.out.5
 
+mpirun -np 4 ./ij -rhsrand -n 15 15 10 -P 2 2 1 -interptype 0 -Pmx 0 -falgout \
+ > interp.out.6
+
 
diff --git a/src/test/TEST_ij/interp.saved b/src/test/TEST_ij/interp.saved
index 73dec7a..e4010af 100644
--- a/src/test/TEST_ij/interp.saved
+++ b/src/test/TEST_ij/interp.saved
@@ -1,42 +1,49 @@
 # Output file: interp.out.0
- Average Convergence Factor = 0.096074
+ Average Convergence Factor = 0.199088
 
-     Complexity:    grid = 1.401333
-                operator = 3.547959
-                   cycle = 7.095850
+     Complexity:    grid = 1.571556
+                operator = 2.973333
+                   cycle = 5.944218
 
 # Output file: interp.out.1
- Average Convergence Factor = 0.096532
+ Average Convergence Factor = 0.197964
 
-     Complexity:    grid = 1.404444
-                operator = 3.456054
-                   cycle = 6.910408
+     Complexity:    grid = 1.580000
+                operator = 2.885102
+                   cycle = 5.765850
 
 # Output file: interp.out.2
- Average Convergence Factor = 0.092304
+ Average Convergence Factor = 0.181048
 
-     Complexity:    grid = 1.400444
-                operator = 3.479660
-                   cycle = 6.958231
+     Complexity:    grid = 1.572889
+                operator = 2.983469
+                   cycle = 5.966327
 
 # Output file: interp.out.3
- Average Convergence Factor = 0.133791
+ Average Convergence Factor = 0.194063
 
-     Complexity:    grid = 1.412889
-                operator = 2.548367
-                   cycle = 5.095034
+     Complexity:    grid = 1.587556
+                operator = 2.684082
+                   cycle = 5.365714
 
 # Output file: interp.out.4
- Average Convergence Factor = 0.139099
+ Average Convergence Factor = 0.195128
 
-     Complexity:    grid = 1.411111
-                operator = 2.540068
-                   cycle = 5.079524
+     Complexity:    grid = 1.588444
+                operator = 2.627483
+                   cycle = 5.252517
 
 # Output file: interp.out.5
- Average Convergence Factor = 0.181513
+ Average Convergence Factor = 0.203482
 
-     Complexity:    grid = 1.408889
-                operator = 2.541224
-                   cycle = 5.080000
+     Complexity:    grid = 1.582667
+                operator = 2.662245
+                   cycle = 5.324218
+
+# Output file: interp.out.6
+ Average Convergence Factor = 0.166090
+
+     Complexity:    grid = 1.713333
+                operator = 3.541020
+                   cycle = 7.080340
 
diff --git a/src/test/TEST_ij/interp.sh b/src/test/TEST_ij/interp.sh
index dd28ecf..0cbc23e 100755
--- a/src/test/TEST_ij/interp.sh
+++ b/src/test/TEST_ij/interp.sh
@@ -24,6 +24,7 @@ FILES="\
  ${TNAME}.out.3\
  ${TNAME}.out.4\
  ${TNAME}.out.5\
+ ${TNAME}.out.6\
 "
 
 for i in $FILES
diff --git a/src/test/TEST_ij/lobpcg.jobs b/src/test/TEST_ij/lobpcg.jobs
index d77ff62..e7ba554 100755
--- a/src/test/TEST_ij/lobpcg.jobs
+++ b/src/test/TEST_ij/lobpcg.jobs
@@ -25,3 +25,39 @@ mpirun -np 2 ./ij -lobpcg -vrand 2 -seed 1 -tol 1.e-4 -vout 1 > lobpcg.out.vout.
 
 # use vectors computed by previous run as the initial
 mpirun -np 2 ./ij -lobpcg -vfromfile -seed 1 > lobpcg.out.vfromfile
+
+#=============================================================================
+# lobpcg: Run default case with all available PCG preconditioners
+#    1: BoomerAMG_PCG
+#    2: DS_PCG
+#    8: ParaSails_PCG
+#   12: Schwarz_PCG
+#   43: Euclid_PCG
+#=============================================================================
+
+
+#PCG run ...LOBPCG run for one eigenpair...LOBPCG for 5 eigenpairs
+mpirun -np 2 ./ij -solver 1 -tol 1.e-6  > lobpcg.out.1.lobpcg
+mpirun -np 2 ./ij -lobpcg -solver 1 -tol 1.e-6 -pcgitr 0 -seed 1 -vrand 1 > lobpcg.out.1.lobpcg.1
+mpirun -np 2 ./ij -lobpcg -solver 1 -tol 1.e-6 -pcgitr 0  -seed 1 -vrand 5 > lobpcg.out.1.lobpcg.5
+
+#PCG run ...LOBPCG run for one eigenpair...LOBPCG for 5 eigenpairs
+mpirun -np 2 ./ij -solver 2 -tol 1.e-6  > lobpcg.out.2.lobpcg
+mpirun -np 2 ./ij -lobpcg -solver 2 -tol 1.e-6  -pcgitr 20 -seed 1 -vrand 1 > lobpcg.out.2.lobpcg.1
+mpirun -np 2 ./ij -lobpcg -solver 2 -tol 1.e-6  -pcgitr 20 -seed 1 -vrand 5 > lobpcg.out.2.lobpcg.5
+
+#PCG run ...LOBPCG run for one eigenpair...LOBPCG for 5 eigenpairs
+mpirun -np 2 ./ij -solver 8 -tol 1.e-6  > lobpcg.out.8.lobpcg
+mpirun -np 2 ./ij -lobpcg -solver 8 -tol 1.e-6  -pcgitr 10 -seed 1 -vrand 1 > lobpcg.out.8.lobpcg.1
+mpirun -np 2 ./ij -lobpcg -solver 8 -tol 1.e-6  -pcgitr 10 -seed 1 -vrand 5 > lobpcg.out.8.lobpcg.5
+
+#PCG run ...LOBPCG run for one eigenpair...LOBPCG for 5 eigenpairs
+mpirun -np 2 ./ij -solver 12 -tol 1.e-6  > lobpcg.out.12.lobpcg
+mpirun -np 2 ./ij -lobpcg -solver 12 -tol 1.e-6  -pcgitr 10 -seed 1 -vrand 1 > lobpcg.out.12.lobpcg.1
+mpirun -np 2 ./ij -lobpcg -solver 12 -tol 1.e-6  -pcgitr 10 -seed 1 -vrand 5 > lobpcg.out.12.lobpcg.5
+
+#PCG run ...LOBPCG run for one eigenpair...LOBPCG for 5 eigenpairs
+mpirun -np 2 ./ij -solver 43 -tol 1.e-6  > lobpcg.out.43.lobpcg
+mpirun -np 2 ./ij -lobpcg -solver 43 -tol 1.e-6  -pcgitr 10 -seed 1 -vrand 1 > lobpcg.out.43.lobpcg.1
+mpirun -np 2 ./ij -lobpcg -solver 43 -tol 1.e-6  -pcgitr 10 -seed 1 -vrand 5 > lobpcg.out.43.lobpcg.5
+
diff --git a/src/test/TEST_ij/lobpcg.saved b/src/test/TEST_ij/lobpcg.saved
index 8c137e8..0465552 100644
--- a/src/test/TEST_ij/lobpcg.saved
+++ b/src/test/TEST_ij/lobpcg.saved
@@ -1,12 +1,112 @@
 # Output file: lobpcg.out.vfromfile
 Eigenvalue lambda   2.43042158313018e-01
-Eigenvalue lambda   4.79521039879659e-01
-Residual   7.81004874790696e-08
-Residual   1.50184070584915e-07
+Eigenvalue lambda   4.79521039879688e-01
+Residual   7.49244273975386e-08
+Residual   3.13511570089364e-07
 
 # Output file: lobpcg.out.vout.1
-Eigenvalue lambda   2.43042158443256e-01
-Eigenvalue lambda   4.79521045718664e-01
-Residual   1.70192416688541e-05
-Residual   8.81917842407598e-05
+Eigenvalue lambda   2.43042159336231e-01
+Eigenvalue lambda   4.79521040556427e-01
+Residual   5.93138904669372e-05
+Residual   3.00153697741728e-05
+
+# Output file: lobpcg.out.1.lobpcg
+Iterations = 6
+Final Relative Residual Norm = 1.767838e-07
+
+# Output file: lobpcg.out.1.lobpcg.1
+Eigenvalue lambda   2.43042158313718e-01
+Residual   1.05888375241797e-06
+
+# Output file: lobpcg.out.1.lobpcg.5
+Eigenvalue lambda   2.43042158313047e-01
+Eigenvalue lambda   4.79521039879736e-01
+Eigenvalue lambda   4.79521039879869e-01
+Eigenvalue lambda   4.79521039880020e-01
+Eigenvalue lambda   7.15999921447659e-01
+Residual   6.11700821392749e-07
+Residual   7.03316483557853e-07
+Residual   8.51096556735168e-07
+Residual   8.72875343951290e-07
+Residual   1.20344285081769e-06
+
+# Output file: lobpcg.out.2.lobpcg
+Iterations = 20
+Final Relative Residual Norm = 5.962015e-07
+
+# Output file: lobpcg.out.2.lobpcg.1
+Eigenvalue lambda   2.43042158313121e-01
+Residual   4.63315030208772e-07
+
+# Output file: lobpcg.out.2.lobpcg.5
+Eigenvalue lambda   2.43042158313016e-01
+Eigenvalue lambda   4.79521039879677e-01
+Eigenvalue lambda   4.79521039879700e-01
+Eigenvalue lambda   4.79521039879780e-01
+Eigenvalue lambda   7.15999921447091e-01
+Residual   1.43322942857248e-07
+Residual   4.86501096019624e-07
+Residual   3.72232026886917e-07
+Residual   5.18126375223749e-07
+Residual   1.02530795888815e-06
+
+# Output file: lobpcg.out.8.lobpcg
+Iterations = 15
+Final Relative Residual Norm = 6.352784e-07
+
+# Output file: lobpcg.out.8.lobpcg.1
+Eigenvalue lambda   2.43042158313109e-01
+Residual   4.35799317942064e-07
+
+# Output file: lobpcg.out.8.lobpcg.5
+Eigenvalue lambda   2.43042158313016e-01
+Eigenvalue lambda   4.79521039879678e-01
+Eigenvalue lambda   4.79521039879699e-01
+Eigenvalue lambda   4.79521039879799e-01
+Eigenvalue lambda   7.15999921446891e-01
+Residual   1.37749965131754e-07
+Residual   4.63010776375628e-07
+Residual   3.87579831562450e-07
+Residual   5.41642190641787e-07
+Residual   9.41630860798197e-07
+
+# Output file: lobpcg.out.12.lobpcg
+Iterations = 13
+Final Relative Residual Norm = 3.896456e-07
+
+# Output file: lobpcg.out.12.lobpcg.1
+Eigenvalue lambda   2.43042158313099e-01
+Residual   3.94665203904632e-07
+
+# Output file: lobpcg.out.12.lobpcg.5
+Eigenvalue lambda   2.43042158313016e-01
+Eigenvalue lambda   4.79521039879675e-01
+Eigenvalue lambda   4.79521039879685e-01
+Eigenvalue lambda   4.79521039879788e-01
+Eigenvalue lambda   7.15999921447147e-01
+Residual   1.36414438599642e-07
+Residual   4.59913522980684e-07
+Residual   3.72152569537697e-07
+Residual   5.07587948203054e-07
+Residual   1.11208056285383e-06
+
+# Output file: lobpcg.out.43.lobpcg
+Iterations = 9
+Final Relative Residual Norm = 5.971754e-07
+
+# Output file: lobpcg.out.43.lobpcg.1
+Eigenvalue lambda   2.43042158313103e-01
+Residual   4.06686999057609e-07
+
+# Output file: lobpcg.out.43.lobpcg.5
+Eigenvalue lambda   2.43042158313016e-01
+Eigenvalue lambda   4.79521039879682e-01
+Eigenvalue lambda   4.79521039879706e-01
+Eigenvalue lambda   4.79521039879825e-01
+Eigenvalue lambda   7.15999921446956e-01
+Residual   1.61376258673867e-07
+Residual   4.76948835732272e-07
+Residual   4.01718127440120e-07
+Residual   5.82137557078099e-07
+Residual   9.97061314149104e-07
 
diff --git a/src/test/TEST_ij/lobpcg.sh b/src/test/TEST_ij/lobpcg.sh
index 25447f7..fe250fd 100755
--- a/src/test/TEST_ij/lobpcg.sh
+++ b/src/test/TEST_ij/lobpcg.sh
@@ -32,6 +32,25 @@ do
   tail -15 $i | head -5
 done > ${TNAME}.out
 
+
+FILES="\
+ ${TNAME}.out.1.lobpcg\
+ ${TNAME}.out.2.lobpcg\
+ ${TNAME}.out.8.lobpcg\
+ ${TNAME}.out.12.lobpcg\
+ ${TNAME}.out.43.lobpcg\
+"
+
+for i in $FILES
+do
+  echo "# Output file: $i"
+  tail -3 $i
+  echo "# Output file: $i.1"
+  tail -13 $i.1 | head -3
+  echo "# Output file: $i.5"
+  tail -21 $i.5 | head -11
+done >> ${TNAME}.out
+
 # Make sure that the output files are reasonable
 CHECK_LINE="Eigenvalue"
 OUT_COUNT=`grep "$CHECK_LINE" ${TNAME}.out | wc -l`
diff --git a/src/test/TEST_ij/matrix.jobs b/src/test/TEST_ij/matrix.jobs
index eb9c52f..d735b68 100755
--- a/src/test/TEST_ij/matrix.jobs
+++ b/src/test/TEST_ij/matrix.jobs
@@ -18,7 +18,7 @@
 
 
 #=============================================================================
-# ij: tests different ways of generating IJMatrix
+# ij -falgout -interptype 0 -Pmx 0: tests different ways of generating IJMatrix
 #=============================================================================
 
 mpirun -np 2 ./ij -rhsrand > matrix.out.0
@@ -27,20 +27,20 @@ mpirun -np 2 ./ij -rhsrand -exact_size > matrix.out.1
 
 mpirun -np 2 ./ij -rhsrand -low_storage > matrix.out.2
 
-mpirun -np 4 ./new_ij -fromfile A_tstoffd -rhsfromfile b_tstoffd > matrix.out.3
+mpirun -np 4 ./ij -fromfile A_tstoffd -rhsfromfile b_tstoffd > matrix.out.3
 
-mpirun -np 2 ./new_ij -test_ij -solver 2 -max_iter 5 -chunk 1 -add 1 > matrix.out.4
+mpirun -np 2 ./ij -test_ij -solver 2 -max_iter 5 -chunk 1 -add 1 > matrix.out.4
 
-mpirun -np 2 ./new_ij -test_ij -solver 2 -max_iter 5 -chunk 1 -add 1 -off_proc 1 > matrix.out.5
+mpirun -np 2 ./ij -test_ij -solver 2 -max_iter 5 -chunk 1 -add 1 -off_proc 1 > matrix.out.5
 
-mpirun -np 2 ./new_ij -test_ij -solver 2 -max_iter 5 -chunk 1 -off_proc 1 > matrix.out.6
+mpirun -np 2 ./ij -test_ij -solver 2 -max_iter 5 -chunk 1 -off_proc 1 > matrix.out.6
 
-mpirun -np 2 ./new_ij -test_ij -solver 2 -max_iter 5 -chunk 1 -off_proc 1 -omp 1 > matrix.out.7
+mpirun -np 2 ./ij -test_ij -solver 2 -max_iter 5 -chunk 1 -off_proc 1 -omp 1 > matrix.out.7
 
-mpirun -np 2 ./new_ij -test_ij -solver 2 -max_iter 5 -chunk 1 -off_proc 1 -omp 1 -add 1 > matrix.out.8
+mpirun -np 2 ./ij -test_ij -solver 2 -max_iter 5 -chunk 1 -off_proc 1 -omp 1 -add 1 > matrix.out.8
 
-mpirun -np 2 ./new_ij -test_ij -solver 2 -max_iter 5 -chunk 1 -omp 1 -exact_size > matrix.out.9
+mpirun -np 2 ./ij -test_ij -solver 2 -max_iter 5 -chunk 1 -omp 1 -exact_size > matrix.out.9
 
-mpirun -np 2 ./new_ij -test_ij -solver 2 -max_iter 5 -chunk 1 -omp 1 > matrix.out.10
+mpirun -np 2 ./ij -test_ij -solver 2 -max_iter 5 -chunk 1 -omp 1 > matrix.out.10
 
-mpirun -np 3 ./new_ij -fromfile test.A -rhsfromfile test.b > matrix.out.11
+mpirun -np 3 ./ij -fromfile test.A -rhsfromfile test.b > matrix.out.11
diff --git a/src/test/TEST_ij/matrix.saved b/src/test/TEST_ij/matrix.saved
index 3113cdd..4812b41 100644
--- a/src/test/TEST_ij/matrix.saved
+++ b/src/test/TEST_ij/matrix.saved
@@ -1,23 +1,23 @@
 # Output file: matrix.out.0
- Average Convergence Factor = 0.017332
+ Average Convergence Factor = 0.173026
 
-     Complexity:    grid = 2.036000
-                operator = 5.803438
-                   cycle = 11.601250
+     Complexity:    grid = 1.555000
+                operator = 2.667344
+                   cycle = 5.332187
 
 # Output file: matrix.out.1
- Average Convergence Factor = 0.017332
+ Average Convergence Factor = 0.173026
 
-     Complexity:    grid = 2.036000
-                operator = 5.803438
-                   cycle = 11.601250
+     Complexity:    grid = 1.555000
+                operator = 2.667344
+                   cycle = 5.332187
 
 # Output file: matrix.out.2
- Average Convergence Factor = 0.017332
+ Average Convergence Factor = 0.173026
 
-     Complexity:    grid = 2.036000
-                operator = 5.803438
-                   cycle = 11.601250
+     Complexity:    grid = 1.555000
+                operator = 2.667344
+                   cycle = 5.332187
 
 # Output file: matrix.out.3
  Average Convergence Factor = 0.000000
@@ -76,9 +76,9 @@
     4    1.558512e+01    0.706405    4.928447e-01
     5    1.111687e+01    0.713300    3.515462e-01
 # Output file: matrix.out.11
- Average Convergence Factor = 0.028399
+ Average Convergence Factor = 0.084659
 
-     Complexity:    grid = 1.260870
-                operator = 1.325581
-                   cycle = 2.325581
+     Complexity:    grid = 1.130435
+                operator = 1.162791
+                   cycle = 2.162791
 
diff --git a/src/test/TEST_ij/options.saved b/src/test/TEST_ij/options.saved
index 4f12759..d961e5b 100644
--- a/src/test/TEST_ij/options.saved
+++ b/src/test/TEST_ij/options.saved
@@ -1,22 +1,22 @@
 # options.out.default
-Eigenvalue lambda   2.43042158313018e-01
-Residual   6.48952190941441e-08
+Eigenvalue lambda   2.43042158313013e-01
+Residual   4.02732923495552e-08
 
 # options.out.no_orthchk
-Eigenvalue lambda   2.43042158313018e-01
-Residual   6.48952190941441e-08
+Eigenvalue lambda   2.43042158313013e-01
+Residual   4.02732923495552e-08
 
 # options.out.pcgitr.0
-Eigenvalue lambda   2.43042158313018e-01
-Residual   6.48952190941441e-08
+Eigenvalue lambda   2.43042158313013e-01
+Residual   4.02732923495552e-08
 
 # options.out.pcgitr.1
-Eigenvalue lambda   2.43042158313018e-01
-Residual   6.48952190941441e-08
+Eigenvalue lambda   2.43042158313013e-01
+Residual   4.02732923495552e-08
 
 # options.out.pcgitr.2
-Eigenvalue lambda   2.43042158313017e-01
-Residual   6.31332863145569e-08
+Eigenvalue lambda   2.43042158313018e-01
+Residual   6.96799149848673e-08
 
 # options.out.pcgtol.01
 Eigenvalue lambda   2.43042158313018e-01
@@ -27,32 +27,32 @@ Eigenvalue lambda   2.43042158313023e-01
 Residual   1.04744967214067e-07
 
 # options.out.seed
-Eigenvalue lambda   2.43042158313018e-01
-Residual   6.48952190941441e-08
+Eigenvalue lambda   2.43042158313013e-01
+Residual   4.02732923495552e-08
 
 # options.out.seed.repeat
-Eigenvalue lambda   2.43042158313018e-01
-Residual   6.48952190941441e-08
+Eigenvalue lambda   2.43042158313013e-01
+Residual   4.02732923495552e-08
 
 # options.out.solver.none
 Eigenvalue lambda   2.43042158316676e-01
 Residual   2.03104260070223e-06
 
 # options.out.verb.1
-Eigenvalue lambda   2.43042158313018e-01
-Residual   6.48952190941441e-08
+Eigenvalue lambda   2.43042158313013e-01
+Residual   4.02732923495552e-08
 
 # options.out.gen.1
 Eigenvalue lambda   1.00000000000000e+00
 Residual   7.54364534145478e-16
 
 # options.out.gen.2
-Eigenvalue lambda   3.14395676813048e-01
-Residual   2.39101583491205e-07
+Eigenvalue lambda   3.14395676812973e-01
+Residual   2.50125397358489e-07
 
 # options.out.orthchk
-Eigenvalue lambda   2.43042158313018e-01
-Residual   6.48952190941441e-08
+Eigenvalue lambda   2.43042158313013e-01
+Residual   4.02732923495552e-08
 
 # options.out.itr.100
 Eigenvalue lambda   1.10912678978531e-01
@@ -67,15 +67,15 @@ Residual   1.68566474191610e+00
 Residual   1.58810659347606e+00
 
 # options.out.vrand.2
-Eigenvalue lambda   2.43042158312873e-01
-Eigenvalue lambda   4.79521039867766e-01
-Residual   2.32087251651523e-07
-Residual   3.75990048555319e-07
+Eigenvalue lambda   2.43042158313021e-01
+Eigenvalue lambda   4.79521039879676e-01
+Residual   1.22006034989775e-07
+Residual   2.33712989952824e-07
 
 # options.out.verb.0
-     Complexity:    grid = 2.036000
-                operator = 5.803438
+     Complexity:    grid = 1.555000
+                operator = 2.667344
 # options.out.verb.2
-Eigenvalue lambda   2.43042158313018e-01
-Residual 6.489522e-08
+Eigenvalue lambda   2.43042158313013e-01
+Residual 4.027329e-08
 =============================================
diff --git a/src/test/TEST_ij/schw.00000 b/src/test/TEST_ij/schw.00000
new file mode 100644
index 0000000..0f2bd1a
--- /dev/null
+++ b/src/test/TEST_ij/schw.00000
@@ -0,0 +1,231 @@
+1 50 1 50
+1 1 4.00000000000000e+00
+1 2 -1.00000000000000e+00
+1 11 -1.00000000000000e+00
+2 2 4.00000000000000e+00
+2 1 -1.00000000000000e+00
+2 3 -1.00000000000000e+00
+2 12 -1.00000000000000e+00
+3 3 4.00000000000000e+00
+3 2 -1.00000000000000e+00
+3 4 -1.00000000000000e+00
+3 13 -1.00000000000000e+00
+4 4 4.00000000000000e+00
+4 3 -1.00000000000000e+00
+4 5 -1.00000000000000e+00
+4 14 -1.00000000000000e+00
+5 5 4.00000000000000e+00
+5 4 -1.00000000000000e+00
+5 6 -1.00000000000000e+00
+5 15 -1.00000000000000e+00
+6 6 4.00000000000000e+00
+6 5 -1.00000000000000e+00
+6 7 -1.00000000000000e+00
+6 16 -1.00000000000000e+00
+7 7 4.00000000000000e+00
+7 6 -1.00000000000000e+00
+7 8 -1.00000000000000e+00
+7 17 -1.00000000000000e+00
+8 8 4.00000000000000e+00
+8 7 -1.00000000000000e+00
+8 9 -1.00000000000000e+00
+8 18 -1.00000000000000e+00
+9 9 4.00000000000000e+00
+9 8 -1.00000000000000e+00
+9 10 -1.00000000000000e+00
+9 19 -1.00000000000000e+00
+10 10 4.00000000000000e+00
+10 9 -1.00000000000000e+00
+10 20 -1.00000000000000e+00
+11 11 4.00000000000000e+00
+11 1 -1.00000000000000e+00
+11 12 -1.00000000000000e+00
+11 21 -1.00000000000000e+00
+12 12 4.00000000000000e+00
+12 2 -1.00000000000000e+00
+12 11 -1.00000000000000e+00
+12 13 -1.00000000000000e+00
+12 22 -1.00000000000000e+00
+13 13 4.00000000000000e+00
+13 3 -1.00000000000000e+00
+13 12 -1.00000000000000e+00
+13 14 -1.00000000000000e+00
+13 23 -1.00000000000000e+00
+14 14 4.00000000000000e+00
+14 4 -1.00000000000000e+00
+14 13 -1.00000000000000e+00
+14 15 -1.00000000000000e+00
+14 24 -1.00000000000000e+00
+15 15 4.00000000000000e+00
+15 5 -1.00000000000000e+00
+15 14 -1.00000000000000e+00
+15 16 -1.00000000000000e+00
+15 25 -1.00000000000000e+00
+16 16 4.00000000000000e+00
+16 6 -1.00000000000000e+00
+16 15 -1.00000000000000e+00
+16 17 -1.00000000000000e+00
+16 26 -1.00000000000000e+00
+17 17 4.00000000000000e+00
+17 7 -1.00000000000000e+00
+17 16 -1.00000000000000e+00
+17 18 -1.00000000000000e+00
+17 27 -1.00000000000000e+00
+18 18 4.00000000000000e+00
+18 8 -1.00000000000000e+00
+18 17 -1.00000000000000e+00
+18 19 -1.00000000000000e+00
+18 28 -1.00000000000000e+00
+19 19 4.00000000000000e+00
+19 9 -1.00000000000000e+00
+19 18 -1.00000000000000e+00
+19 20 -1.00000000000000e+00
+19 29 -1.00000000000000e+00
+20 20 4.00000000000000e+00
+20 10 -1.00000000000000e+00
+20 19 -1.00000000000000e+00
+20 30 -1.00000000000000e+00
+21 21 4.00000000000000e+00
+21 11 -1.00000000000000e+00
+21 22 -1.00000000000000e+00
+21 31 -1.00000000000000e+00
+22 22 4.00000000000000e+00
+22 12 -1.00000000000000e+00
+22 21 -1.00000000000000e+00
+22 23 -1.00000000000000e+00
+22 32 -1.00000000000000e+00
+23 23 4.00000000000000e+00
+23 13 -1.00000000000000e+00
+23 22 -1.00000000000000e+00
+23 24 -1.00000000000000e+00
+23 33 -1.00000000000000e+00
+24 24 4.00000000000000e+00
+24 14 -1.00000000000000e+00
+24 23 -1.00000000000000e+00
+24 25 -1.00000000000000e+00
+24 34 -1.00000000000000e+00
+25 25 4.00000000000000e+00
+25 15 -1.00000000000000e+00
+25 24 -1.00000000000000e+00
+25 26 -1.00000000000000e+00
+25 35 -1.00000000000000e+00
+26 26 4.00000000000000e+00
+26 16 -1.00000000000000e+00
+26 25 -1.00000000000000e+00
+26 27 -1.00000000000000e+00
+26 36 -1.00000000000000e+00
+27 27 4.00000000000000e+00
+27 17 -1.00000000000000e+00
+27 26 -1.00000000000000e+00
+27 28 -1.00000000000000e+00
+27 37 -1.00000000000000e+00
+28 28 4.00000000000000e+00
+28 18 -1.00000000000000e+00
+28 27 -1.00000000000000e+00
+28 29 -1.00000000000000e+00
+28 38 -1.00000000000000e+00
+29 29 4.00000000000000e+00
+29 19 -1.00000000000000e+00
+29 28 -1.00000000000000e+00
+29 30 -1.00000000000000e+00
+29 39 -1.00000000000000e+00
+30 30 4.00000000000000e+00
+30 20 -1.00000000000000e+00
+30 29 -1.00000000000000e+00
+30 40 -1.00000000000000e+00
+31 31 4.00000000000000e+00
+31 21 -1.00000000000000e+00
+31 32 -1.00000000000000e+00
+31 41 -1.00000000000000e+00
+32 32 4.00000000000000e+00
+32 22 -1.00000000000000e+00
+32 31 -1.00000000000000e+00
+32 33 -1.00000000000000e+00
+32 42 -1.00000000000000e+00
+33 33 4.00000000000000e+00
+33 23 -1.00000000000000e+00
+33 32 -1.00000000000000e+00
+33 34 -1.00000000000000e+00
+33 43 -1.00000000000000e+00
+34 34 4.00000000000000e+00
+34 24 -1.00000000000000e+00
+34 33 -1.00000000000000e+00
+34 35 -1.00000000000000e+00
+34 44 -1.00000000000000e+00
+35 35 4.00000000000000e+00
+35 25 -1.00000000000000e+00
+35 34 -1.00000000000000e+00
+35 36 -1.00000000000000e+00
+35 45 -1.00000000000000e+00
+36 36 4.00000000000000e+00
+36 26 -1.00000000000000e+00
+36 35 -1.00000000000000e+00
+36 37 -1.00000000000000e+00
+36 46 -1.00000000000000e+00
+37 37 4.00000000000000e+00
+37 27 -1.00000000000000e+00
+37 36 -1.00000000000000e+00
+37 38 -1.00000000000000e+00
+37 47 -1.00000000000000e+00
+38 38 4.00000000000000e+00
+38 28 -1.00000000000000e+00
+38 37 -1.00000000000000e+00
+38 39 -1.00000000000000e+00
+38 48 -1.00000000000000e+00
+39 39 4.00000000000000e+00
+39 29 -1.00000000000000e+00
+39 38 -1.00000000000000e+00
+39 40 -1.00000000000000e+00
+39 49 -1.00000000000000e+00
+40 40 4.00000000000000e+00
+40 30 -1.00000000000000e+00
+40 39 -1.00000000000000e+00
+40 50 -1.00000000000000e+00
+41 41 4.00000000000000e+00
+41 31 -1.00000000000000e+00
+41 42 -1.00000000000000e+00
+41 51 -1.00000000000000e+00
+42 42 4.00000000000000e+00
+42 32 -1.00000000000000e+00
+42 41 -1.00000000000000e+00
+42 43 -1.00000000000000e+00
+42 52 -1.00000000000000e+00
+43 43 4.00000000000000e+00
+43 33 -1.00000000000000e+00
+43 42 -1.00000000000000e+00
+43 44 -1.00000000000000e+00
+43 53 -1.00000000000000e+00
+44 44 4.00000000000000e+00
+44 34 -1.00000000000000e+00
+44 43 -1.00000000000000e+00
+44 45 -1.00000000000000e+00
+44 54 -1.00000000000000e+00
+45 45 4.00000000000000e+00
+45 35 -1.00000000000000e+00
+45 44 -1.00000000000000e+00
+45 46 -1.00000000000000e+00
+45 55 -1.00000000000000e+00
+46 46 4.00000000000000e+00
+46 36 -1.00000000000000e+00
+46 45 -1.00000000000000e+00
+46 47 -1.00000000000000e+00
+46 56 -1.00000000000000e+00
+47 47 4.00000000000000e+00
+47 37 -1.00000000000000e+00
+47 46 -1.00000000000000e+00
+47 48 -1.00000000000000e+00
+47 57 -1.00000000000000e+00
+48 48 4.00000000000000e+00
+48 38 -1.00000000000000e+00
+48 47 -1.00000000000000e+00
+48 49 -1.00000000000000e+00
+48 58 -1.00000000000000e+00
+49 49 4.00000000000000e+00
+49 39 -1.00000000000000e+00
+49 48 -1.00000000000000e+00
+49 50 -1.00000000000000e+00
+49 59 -1.00000000000000e+00
+50 50 4.00000000000000e+00
+50 40 -1.00000000000000e+00
+50 49 -1.00000000000000e+00
+50 60 -1.00000000000000e+00
diff --git a/src/test/TEST_ij/schw.00001 b/src/test/TEST_ij/schw.00001
new file mode 100644
index 0000000..0d84a3a
--- /dev/null
+++ b/src/test/TEST_ij/schw.00001
@@ -0,0 +1 @@
+51 50 51 50
diff --git a/src/test/TEST_ij/schw.00002 b/src/test/TEST_ij/schw.00002
new file mode 100644
index 0000000..f7dea5d
--- /dev/null
+++ b/src/test/TEST_ij/schw.00002
@@ -0,0 +1,231 @@
+51 100 51 100
+51 51 4.00000000000000e+00
+51 52 -1.00000000000000e+00
+51 61 -1.00000000000000e+00
+51 41 -1.00000000000000e+00
+52 52 4.00000000000000e+00
+52 51 -1.00000000000000e+00
+52 53 -1.00000000000000e+00
+52 62 -1.00000000000000e+00
+52 42 -1.00000000000000e+00
+53 53 4.00000000000000e+00
+53 52 -1.00000000000000e+00
+53 54 -1.00000000000000e+00
+53 63 -1.00000000000000e+00
+53 43 -1.00000000000000e+00
+54 54 4.00000000000000e+00
+54 53 -1.00000000000000e+00
+54 55 -1.00000000000000e+00
+54 64 -1.00000000000000e+00
+54 44 -1.00000000000000e+00
+55 55 4.00000000000000e+00
+55 54 -1.00000000000000e+00
+55 56 -1.00000000000000e+00
+55 65 -1.00000000000000e+00
+55 45 -1.00000000000000e+00
+56 56 4.00000000000000e+00
+56 55 -1.00000000000000e+00
+56 57 -1.00000000000000e+00
+56 66 -1.00000000000000e+00
+56 46 -1.00000000000000e+00
+57 57 4.00000000000000e+00
+57 56 -1.00000000000000e+00
+57 58 -1.00000000000000e+00
+57 67 -1.00000000000000e+00
+57 47 -1.00000000000000e+00
+58 58 4.00000000000000e+00
+58 57 -1.00000000000000e+00
+58 59 -1.00000000000000e+00
+58 68 -1.00000000000000e+00
+58 48 -1.00000000000000e+00
+59 59 4.00000000000000e+00
+59 58 -1.00000000000000e+00
+59 60 -1.00000000000000e+00
+59 69 -1.00000000000000e+00
+59 49 -1.00000000000000e+00
+60 60 4.00000000000000e+00
+60 59 -1.00000000000000e+00
+60 70 -1.00000000000000e+00
+60 50 -1.00000000000000e+00
+61 61 4.00000000000000e+00
+61 51 -1.00000000000000e+00
+61 62 -1.00000000000000e+00
+61 71 -1.00000000000000e+00
+62 62 4.00000000000000e+00
+62 52 -1.00000000000000e+00
+62 61 -1.00000000000000e+00
+62 63 -1.00000000000000e+00
+62 72 -1.00000000000000e+00
+63 63 4.00000000000000e+00
+63 53 -1.00000000000000e+00
+63 62 -1.00000000000000e+00
+63 64 -1.00000000000000e+00
+63 73 -1.00000000000000e+00
+64 64 4.00000000000000e+00
+64 54 -1.00000000000000e+00
+64 63 -1.00000000000000e+00
+64 65 -1.00000000000000e+00
+64 74 -1.00000000000000e+00
+65 65 4.00000000000000e+00
+65 55 -1.00000000000000e+00
+65 64 -1.00000000000000e+00
+65 66 -1.00000000000000e+00
+65 75 -1.00000000000000e+00
+66 66 4.00000000000000e+00
+66 56 -1.00000000000000e+00
+66 65 -1.00000000000000e+00
+66 67 -1.00000000000000e+00
+66 76 -1.00000000000000e+00
+67 67 4.00000000000000e+00
+67 57 -1.00000000000000e+00
+67 66 -1.00000000000000e+00
+67 68 -1.00000000000000e+00
+67 77 -1.00000000000000e+00
+68 68 4.00000000000000e+00
+68 58 -1.00000000000000e+00
+68 67 -1.00000000000000e+00
+68 69 -1.00000000000000e+00
+68 78 -1.00000000000000e+00
+69 69 4.00000000000000e+00
+69 59 -1.00000000000000e+00
+69 68 -1.00000000000000e+00
+69 70 -1.00000000000000e+00
+69 79 -1.00000000000000e+00
+70 70 4.00000000000000e+00
+70 60 -1.00000000000000e+00
+70 69 -1.00000000000000e+00
+70 80 -1.00000000000000e+00
+71 71 4.00000000000000e+00
+71 61 -1.00000000000000e+00
+71 72 -1.00000000000000e+00
+71 81 -1.00000000000000e+00
+72 72 4.00000000000000e+00
+72 62 -1.00000000000000e+00
+72 71 -1.00000000000000e+00
+72 73 -1.00000000000000e+00
+72 82 -1.00000000000000e+00
+73 73 4.00000000000000e+00
+73 63 -1.00000000000000e+00
+73 72 -1.00000000000000e+00
+73 74 -1.00000000000000e+00
+73 83 -1.00000000000000e+00
+74 74 4.00000000000000e+00
+74 64 -1.00000000000000e+00
+74 73 -1.00000000000000e+00
+74 75 -1.00000000000000e+00
+74 84 -1.00000000000000e+00
+75 75 4.00000000000000e+00
+75 65 -1.00000000000000e+00
+75 74 -1.00000000000000e+00
+75 76 -1.00000000000000e+00
+75 85 -1.00000000000000e+00
+76 76 4.00000000000000e+00
+76 66 -1.00000000000000e+00
+76 75 -1.00000000000000e+00
+76 77 -1.00000000000000e+00
+76 86 -1.00000000000000e+00
+77 77 4.00000000000000e+00
+77 67 -1.00000000000000e+00
+77 76 -1.00000000000000e+00
+77 78 -1.00000000000000e+00
+77 87 -1.00000000000000e+00
+78 78 4.00000000000000e+00
+78 68 -1.00000000000000e+00
+78 77 -1.00000000000000e+00
+78 79 -1.00000000000000e+00
+78 88 -1.00000000000000e+00
+79 79 4.00000000000000e+00
+79 69 -1.00000000000000e+00
+79 78 -1.00000000000000e+00
+79 80 -1.00000000000000e+00
+79 89 -1.00000000000000e+00
+80 80 4.00000000000000e+00
+80 70 -1.00000000000000e+00
+80 79 -1.00000000000000e+00
+80 90 -1.00000000000000e+00
+81 81 4.00000000000000e+00
+81 71 -1.00000000000000e+00
+81 82 -1.00000000000000e+00
+81 91 -1.00000000000000e+00
+82 82 4.00000000000000e+00
+82 72 -1.00000000000000e+00
+82 81 -1.00000000000000e+00
+82 83 -1.00000000000000e+00
+82 92 -1.00000000000000e+00
+83 83 4.00000000000000e+00
+83 73 -1.00000000000000e+00
+83 82 -1.00000000000000e+00
+83 84 -1.00000000000000e+00
+83 93 -1.00000000000000e+00
+84 84 4.00000000000000e+00
+84 74 -1.00000000000000e+00
+84 83 -1.00000000000000e+00
+84 85 -1.00000000000000e+00
+84 94 -1.00000000000000e+00
+85 85 4.00000000000000e+00
+85 75 -1.00000000000000e+00
+85 84 -1.00000000000000e+00
+85 86 -1.00000000000000e+00
+85 95 -1.00000000000000e+00
+86 86 4.00000000000000e+00
+86 76 -1.00000000000000e+00
+86 85 -1.00000000000000e+00
+86 87 -1.00000000000000e+00
+86 96 -1.00000000000000e+00
+87 87 4.00000000000000e+00
+87 77 -1.00000000000000e+00
+87 86 -1.00000000000000e+00
+87 88 -1.00000000000000e+00
+87 97 -1.00000000000000e+00
+88 88 4.00000000000000e+00
+88 78 -1.00000000000000e+00
+88 87 -1.00000000000000e+00
+88 89 -1.00000000000000e+00
+88 98 -1.00000000000000e+00
+89 89 4.00000000000000e+00
+89 79 -1.00000000000000e+00
+89 88 -1.00000000000000e+00
+89 90 -1.00000000000000e+00
+89 99 -1.00000000000000e+00
+90 90 4.00000000000000e+00
+90 80 -1.00000000000000e+00
+90 89 -1.00000000000000e+00
+90 100 -1.00000000000000e+00
+91 91 4.00000000000000e+00
+91 81 -1.00000000000000e+00
+91 92 -1.00000000000000e+00
+92 92 4.00000000000000e+00
+92 82 -1.00000000000000e+00
+92 91 -1.00000000000000e+00
+92 93 -1.00000000000000e+00
+93 93 4.00000000000000e+00
+93 83 -1.00000000000000e+00
+93 92 -1.00000000000000e+00
+93 94 -1.00000000000000e+00
+94 94 4.00000000000000e+00
+94 84 -1.00000000000000e+00
+94 93 -1.00000000000000e+00
+94 95 -1.00000000000000e+00
+95 95 4.00000000000000e+00
+95 85 -1.00000000000000e+00
+95 94 -1.00000000000000e+00
+95 96 -1.00000000000000e+00
+96 96 4.00000000000000e+00
+96 86 -1.00000000000000e+00
+96 95 -1.00000000000000e+00
+96 97 -1.00000000000000e+00
+97 97 4.00000000000000e+00
+97 87 -1.00000000000000e+00
+97 96 -1.00000000000000e+00
+97 98 -1.00000000000000e+00
+98 98 4.00000000000000e+00
+98 88 -1.00000000000000e+00
+98 97 -1.00000000000000e+00
+98 99 -1.00000000000000e+00
+99 99 4.00000000000000e+00
+99 89 -1.00000000000000e+00
+99 98 -1.00000000000000e+00
+99 100 -1.00000000000000e+00
+100 100 4.00000000000000e+00
+100 90 -1.00000000000000e+00
+100 99 -1.00000000000000e+00
diff --git a/src/test/TEST_ij/smoother.jobs b/src/test/TEST_ij/smoother.jobs
index 1c7ad38..bd85133 100755
--- a/src/test/TEST_ij/smoother.jobs
+++ b/src/test/TEST_ij/smoother.jobs
@@ -18,40 +18,72 @@
 
 
 #=============================================================================
-# ij: Run default case with different coarsenings, hybrid GS,
+# ij: Run default case with different smoothers
 #    1: hybrid SOR on 3 procs
 #    2: hybrid scaled symGS on 4 procs. with scaled opt. outer wt on fine level
 #    3: hybrid GS on 4 procs with backward solve GS on up cycle
 #    4: hybrid GS, no CF-relaxation, on 4 procs
 #    5: hybrid SOR on 4 procs, determine a weight automatically
-#    6: use Schwarz smoothing
-#    7: use Schwarz smoothing with automatic weight determination
+#    6: use Schwarz smoothing on first level 
+#    7: use Schwarz smoothing on first level with automatic weight determination
 #    8: use 3 level Schwarz smoothing with automatic weight determination
 #       followed by SSOR with automatic weight determination
+#    9: CF Jacobi on 3 procs
+#   10: CF L1 Jacobi on 3 procs
+#   11: L1 Jacobi on 3 procs
+#   12: L1 Gauss-Seidel on 4 procs
+#   13: Polynomial (Chebyshev 2nd order) on 4 procs
+#   14: Polynomial (Chebyshev 3rd order) on 4 procs
+#   15: FCF Jacobi on 4 procs
+#   16: CG smoother on 4 procs
 #=============================================================================
 
-mpirun -np 3 ./new_ij -rhsrand -n 15 30 10 -w 1.1 -owl 1.0 0 \
+mpirun -np 3 ./ij -rhsrand -n 15 30 10 -w 1.1 -owl 1.0 0 \
  > smoother.out.0
 
-mpirun -np 4 ./new_ij -rhsrand -n 15 15 10 -P 2 2 1 -rlx 6 -owl -10 0 \
+mpirun -np 4 ./ij -rhsrand -n 15 15 10 -P 2 2 1 -rlx 6 -owl -10 0 \
  > smoother.out.1
 
-mpirun -np 4 ./new_ij -rhsrand -n 20 20 10 -P 2 2 1 -rlx_up 4 \
+mpirun -np 4 ./ij -rhsrand -n 20 20 10 -P 2 2 1 -rlx_up 4 \
  > smoother.out.2
 
-mpirun -np 4 ./new_ij -rhsrand -n 20 20 10 -P 2 2 1 -CF 0 \
+mpirun -np 4 ./ij -rhsrand -n 20 20 10 -P 2 2 1 -CF 1 \
  > smoother.out.3
 
-mpirun -np 4 ./new_ij -rhsrand -solver 1 -rlx 6 -n 20 20 10 -P 2 2 1 -CF 0 \
+mpirun -np 4 ./ij -rhsrand -solver 1 -rlx 6 -n 20 20 10 -P 2 2 1 \
 -w -10 > smoother.out.4
 
-mpirun -np 4 ./new_ij -rhsrand -solver 1 -rlx 6 -n 20 20 10 -P 2 2 1 \
--sysL 2 -nf 2 -smlv 1 -smtype 6 -dom 1 -ov 0 -var 1 > smoother.out.5
+mpirun -np 3 ./ij -rhsrand -fromfile schw -agg_nl 1 -smlv 1 > smoother.out.5
 
-mpirun -np 4 ./new_ij -rhsrand -solver 1 -rlx 6 -n 20 20 10 -P 2 2 1 \
+mpirun -np 4 ./ij -rhsrand -solver 1 -rlx 6 -n 20 20 10 -P 2 2 1 \
 -sysL 2 -nf 2 -smlv 1 -smtype 6 -dom 1 -ov 0 -var 1 -ow -10 -sw -10 \
 > smoother.out.6
 
-mpirun -np 4 ./new_ij -rlx 6 -n 20 20 10 -P 2 2 1 -sysL 3 -nf 3 -smlv 3 \
+mpirun -np 4 ./ij -rlx 6 -n 20 20 10 -P 2 2 1 -sysL 3 -nf 3 -smlv 3 \
 -smtype 6 -dom 1 -ov 0 -w -10 -sw -10 \
 > smoother.out.7
+
+mpirun -np 3 ./ij -rhsrand -n 15 30 10 -rlx 0 -CF 1 \
+> smoother.out.8
+
+mpirun -np 3 ./ij -rhsrand -n 15 30 10 -rlx 18 -CF 1 \
+> smoother.out.9
+
+mpirun -np 3 ./ij -rhsrand -n 15 30 10 -rlx 18 \
+> smoother.out.10
+
+mpirun -np 4 ./ij -rhsrand -solver 1 -rlx 8 -n 20 20 10 -P 2 2 1 \
+> smoother.out.11
+
+mpirun -np 4 ./ij -rhsrand -solver 1 -rlx 16 -n 20 20 10 -P 2 2 1 \
+> smoother.out.12
+
+mpirun -np 4 ./ij -rhsrand -solver 1 -rlx 16 -cheby_order 3 -n 20 20 10 \
+ -P 2 2 1 > smoother.out.13
+
+mpirun -np 4 ./ij -rhsrand -solver 1 -rlx 17 -n 20 20 10 -P 2 2 1 \
+> smoother.out.14
+
+mpirun -np 4 ./ij -rhsrand -solver 1 -rlx 15 -n 20 20 10 -P 2 2 1 \
+> smoother.out.15
+
diff --git a/src/test/TEST_ij/smoother.saved b/src/test/TEST_ij/smoother.saved
index 3aa1069..914e63b 100644
--- a/src/test/TEST_ij/smoother.saved
+++ b/src/test/TEST_ij/smoother.saved
@@ -1,44 +1,76 @@
 # Output file: smoother.out.0
- Average Convergence Factor = 0.016690
+ Average Convergence Factor = 0.239197
 
-     Complexity:    grid = 1.692000
-                operator = 3.517576
-                   cycle = 7.032997
+     Complexity:    grid = 1.608444
+                operator = 2.760673
+                   cycle = 5.520808
 
 # Output file: smoother.out.1
- Average Convergence Factor = 0.021100
+ Average Convergence Factor = 0.114249
 
-     Complexity:    grid = 1.713333
-                operator = 3.541020
-                   cycle = 7.080340
+     Complexity:    grid = 1.587556
+                operator = 2.684082
+                   cycle = 5.365714
 
 # Output file: smoother.out.2
- Average Convergence Factor = 0.028763
+ Average Convergence Factor = 0.200420
 
-     Complexity:    grid = 1.695000
-                operator = 3.591288
-                   cycle = 7.180720
+     Complexity:    grid = 1.601000
+                operator = 2.738485
+                   cycle = 5.476364
 
 # Output file: smoother.out.3
- Average Convergence Factor = 0.125158
+ Average Convergence Factor = 0.120196
 
-     Complexity:    grid = 1.695000
-                operator = 3.591288
-                   cycle = 7.180720
+     Complexity:    grid = 1.601000
+                operator = 2.738485
+                   cycle = 5.476364
 
 # Output file: smoother.out.4
-Iterations = 5
-Final Relative Residual Norm = 9.891191e-09
+Iterations = 6
+Final Relative Residual Norm = 5.846604e-09
 
 # Output file: smoother.out.5
-Iterations = 36
-Final Relative Residual Norm = 9.189342e-09
+BoomerAMG Iterations = 17
+Final Relative Residual Norm = 5.893051e-09
 
 # Output file: smoother.out.6
-Iterations = 8
-Final Relative Residual Norm = 8.817485e-10
+Iterations = 10
+Final Relative Residual Norm = 5.359205e-09
 
 # Output file: smoother.out.7
+BoomerAMG Iterations = 18
+Final Relative Residual Norm = 7.334920e-09
+
+# Output file: smoother.out.8
+BoomerAMG Iterations = 11
+Final Relative Residual Norm = 7.457693e-09
+
+# Output file: smoother.out.9
 BoomerAMG Iterations = 14
-Final Relative Residual Norm = 3.901903e-09
+Final Relative Residual Norm = 4.118037e-09
+
+# Output file: smoother.out.10
+BoomerAMG Iterations = 23
+Final Relative Residual Norm = 8.254191e-09
+
+# Output file: smoother.out.11
+Iterations = 6
+Final Relative Residual Norm = 2.509163e-09
+
+# Output file: smoother.out.12
+Iterations = 6
+Final Relative Residual Norm = 2.510138e-09
+
+# Output file: smoother.out.13
+Iterations = 5
+Final Relative Residual Norm = 6.702216e-09
+
+# Output file: smoother.out.14
+Iterations = 6
+Final Relative Residual Norm = 5.044385e-10
+
+# Output file: smoother.out.15
+Iterations = 15
+Final Relative Residual Norm = 5.807749e-09
 
diff --git a/src/test/TEST_ij/smoother.sh b/src/test/TEST_ij/smoother.sh
index 0f840a5..35b3fe5 100755
--- a/src/test/TEST_ij/smoother.sh
+++ b/src/test/TEST_ij/smoother.sh
@@ -35,6 +35,14 @@ FILES="\
  ${TNAME}.out.5\
  ${TNAME}.out.6\
  ${TNAME}.out.7\
+ ${TNAME}.out.8\
+ ${TNAME}.out.9\
+ ${TNAME}.out.10\
+ ${TNAME}.out.11\
+ ${TNAME}.out.12\
+ ${TNAME}.out.13\
+ ${TNAME}.out.14\
+ ${TNAME}.out.15\
 "
 
 for i in $FILES
diff --git a/src/test/TEST_ij/solvers.jobs b/src/test/TEST_ij/solvers.jobs
index 3f9785e..ac2aee8 100755
--- a/src/test/TEST_ij/solvers.jobs
+++ b/src/test/TEST_ij/solvers.jobs
@@ -32,14 +32,7 @@
 
 
 
-# lobpcg: Run default case with all available PCG preconditioners
-#    1: BoomerAMG_PCG
-#    2: DS_PCG
-#    8: ParaSails_PCG
-#   12: Schwarz_PCG
-#   43: Euclid_PCG
-#
-# new_ij: test systems AMG
+# ij: test systems AMG
 #	  unknown approach
 #	  hybrid approach with block smoother
 # 	  nodal approach
@@ -64,62 +57,36 @@ mpirun -np 2 ./ij -solver 20 -cf 0.5 -rhsrand > solvers.out.9
 mpirun -np 2 ./ij -solver 20 -cf 0.5 -rhsrand -solver_type 2 > solvers.out.10
 mpirun -np 2 ./ij -solver 20 -cf 0.5 -rhsrand -solver_type 3 > solvers.out.11
 
-
-#PCG run ...LOBPCG run for one eigenpair...LOBPCG for 5 eigenpairs
-mpirun -np 2 ./ij -solver 1 -tol 1.e-6  > solvers.out.1.lobpcg
-mpirun -np 2 ./ij -lobpcg -solver 1 -tol 1.e-6 -pcgitr 0 -seed 1 -vrand 1 > solvers.out.1.lobpcg.1
-mpirun -np 2 ./ij -lobpcg -solver 1 -tol 1.e-6 -pcgitr 0  -seed 1 -vrand 5 > solvers.out.1.lobpcg.5
-
-#PCG run ...LOBPCG run for one eigenpair...LOBPCG for 5 eigenpairs
-mpirun -np 2 ./ij -solver 2 -tol 1.e-6  > solvers.out.2.lobpcg
-mpirun -np 2 ./ij -lobpcg -solver 2 -tol 1.e-6  -pcgitr 20 -seed 1 -vrand 1 > solvers.out.2.lobpcg.1
-mpirun -np 2 ./ij -lobpcg -solver 2 -tol 1.e-6  -pcgitr 20 -seed 1 -vrand 5 > solvers.out.2.lobpcg.5
-
-#PCG run ...LOBPCG run for one eigenpair...LOBPCG for 5 eigenpairs
-mpirun -np 2 ./ij -solver 8 -tol 1.e-6  > solvers.out.8.lobpcg
-mpirun -np 2 ./ij -lobpcg -solver 8 -tol 1.e-6  -pcgitr 10 -seed 1 -vrand 1 > solvers.out.8.lobpcg.1
-mpirun -np 2 ./ij -lobpcg -solver 8 -tol 1.e-6  -pcgitr 10 -seed 1 -vrand 5 > solvers.out.8.lobpcg.5
-
-#PCG run ...LOBPCG run for one eigenpair...LOBPCG for 5 eigenpairs
-mpirun -np 2 ./ij -solver 12 -tol 1.e-6  > solvers.out.12.lobpcg
-mpirun -np 2 ./ij -lobpcg -solver 12 -tol 1.e-6  -pcgitr 10 -seed 1 -vrand 1 > solvers.out.12.lobpcg.1
-mpirun -np 2 ./ij -lobpcg -solver 12 -tol 1.e-6  -pcgitr 10 -seed 1 -vrand 5 > solvers.out.12.lobpcg.5
-
-#PCG run ...LOBPCG run for one eigenpair...LOBPCG for 5 eigenpairs
-mpirun -np 2 ./ij -solver 43 -tol 1.e-6  > solvers.out.43.lobpcg
-mpirun -np 2 ./ij -lobpcg -solver 43 -tol 1.e-6  -pcgitr 10 -seed 1 -vrand 1 > solvers.out.43.lobpcg.1
-mpirun -np 2 ./ij -lobpcg -solver 43 -tol 1.e-6  -pcgitr 10 -seed 1 -vrand 5 > solvers.out.43.lobpcg.5
-
 #systems AMG run ...unknown approach, hybrid approach, nodal approach
-mpirun -np 2 ./new_ij -n 20 20 20 -sysL 2 -nf 2 > solvers.out.sysu
-mpirun -np 2 ./new_ij -n 20 20 20 -sysL 2 -nf 2 -nodal 1 -smtype 6 -smlv 10 -dom 1 -ov 0 > solvers.out.sysh
-mpirun -np 2 ./new_ij -n 20 20 20 -sysL 2 -nf 2 -interptype 10 > solvers.out.sysn
+mpirun -np 2 ./ij -n 20 20 20 -sysL 2 -nf 2 > solvers.out.sysu
+mpirun -np 2 ./ij -n 20 20 20 -sysL 2 -nf 2 -nodal 1 -smtype 6 -smlv 10 -dom 1 -ov 0 > solvers.out.sysh
+mpirun -np 2 ./ij -n 20 20 20 -sysL 2 -nf 2 -interptype 10 -Pmx 6 > solvers.out.sysn
 
 #LGMRS and FlexGMRES
-mpirun -np 2 ./new_ij -solver 50 -rhsrand > solvers.out.101
-mpirun -np 2 ./new_ij -solver 51 -rhsrand > solvers.out.102
-mpirun -np 2 ./new_ij -solver 60 -rhsrand > solvers.out.103
-mpirun -np 2 ./new_ij -solver 61 -rhsrand > solvers.out.104
+mpirun -np 2 ./ij -solver 50 -rhsrand > solvers.out.101
+mpirun -np 2 ./ij -solver 51 -rhsrand > solvers.out.102
+mpirun -np 2 ./ij -solver 60 -rhsrand > solvers.out.103
+mpirun -np 2 ./ij -solver 61 -rhsrand > solvers.out.104
 
 #agglomerated coarse grid solve
-mpirun -np 8 ./new_ij -n 80 80 80 -P 2 2 2 -agg_nl 1 -hmis -interptype 6 -Pmx 4 -seq_th 100 -solver 1 -rlx 6 > solvers.out.105
-mpirun -np 8 ./new_ij -n 40 40 40 -P 2 2 2 -agg_nl 1 -hmis -interptype 6 -Pmx 4 -seq_th 100 -solver 1 -rlx 6 -sysL 3 -nf 3 > solvers.out.107
+mpirun -np 8 ./ij -n 80 80 80 -P 2 2 2 -agg_nl 1 -seq_th 100 -solver 1 -rlx 6 > solvers.out.105
+mpirun -np 8 ./ij -n 40 40 40 -P 2 2 2 -agg_nl 1 -seq_th 100 -solver 1 -rlx 6 -sysL 3 -nf 3 > solvers.out.107
 
 #redundant coarse grid solve
-mpirun -np 8 ./new_ij -n 80 80 80 -P 2 2 2 -agg_nl 1 -hmis -interptype 6 -Pmx 4 -seq_th 100 -solver 1 -rlx 6 -red 1 > solvers.out.106
-mpirun -np 8 ./new_ij -n 40 40 40 -P 2 2 2 -agg_nl 1 -hmis -interptype 6 -Pmx 4 -seq_th 100 -solver 1 -rlx 6 -sysL 3 -nf 3 -red 1 > solvers.out.108
+mpirun -np 8 ./ij -n 80 80 80 -P 2 2 2 -agg_nl 1 -seq_th 100 -solver 1 -rlx 6 -red 1 > solvers.out.106
+mpirun -np 8 ./ij -n 40 40 40 -P 2 2 2 -agg_nl 1 -seq_th 100 -solver 1 -rlx 6 -sysL 3 -nf 3 -red 1 > solvers.out.108
 
 #additive cycles
-mpirun -np 2 ./new_ij -n 20 20 20 -P 2 1 1 -hmis -interptype 6 -Pmx 4 -agg_nl 1 -solver 1 -CF 0 -rlx 0 -w 0.7 -rlx_coarse 0 -ns_coarse 2 > solvers.out.109
-mpirun -np 2 ./new_ij -n 20 20 20 -P 2 1 1 -hmis -interptype 6 -Pmx 4 -agg_nl 1 -solver 1 -CF 0 -rlx 0 -w 0.7 -mult_add 0 > solvers.out.110
-mpirun -np 4 ./new_ij -n 20 20 20 -P 2 2 1 -hmis -interptype 6 -Pmx 4 -agg_nl 1 -solver 1 -simple 0 > solvers.out.111
-mpirun -np 8 ./new_ij -n 20 20 20 -P 2 2 2 -hmis -interptype 6 -Pmx 4 -agg_nl 1 -solver 3 -additive 1 > solvers.out.112
-mpirun -np 8 ./new_ij -n 20 20 20 -P 2 2 2 -hmis -interptype 6 -Pmx 4 -agg_nl 1 -solver 3 -mult_add 0 -add_Pmx 5 > solvers.out.113
+mpirun -np 2 ./ij -n 20 20 20 -P 2 1 1 -agg_nl 1 -solver 1 -CF 0 -rlx 0 -w 0.7 -rlx_coarse 0 -ns_coarse 2 > solvers.out.109
+mpirun -np 2 ./ij -n 20 20 20 -P 2 1 1 -agg_nl 1 -solver 1 -CF 0 -add_rlx 0 -add_w 0.7 -mult_add 0 > solvers.out.110
+mpirun -np 4 ./ij -n 20 20 20 -P 2 2 1 -agg_nl 1 -solver 1 -simple 0 > solvers.out.111
+mpirun -np 8 ./ij -n 20 20 20 -P 2 2 2 -agg_nl 1 -solver 3 -additive 1 > solvers.out.112
+mpirun -np 8 ./ij -n 20 20 20 -P 2 2 2 -agg_nl 1 -solver 3 -mult_add 0 -add_Pmx 5 > solvers.out.113
 
 #nonGalerkin version
-mpirun -np 8 ./new_ij -n 40 40 40 -P 2 2 2 -hmis -interptype 6 -Pmx 4 -nongalerk_tol 1 0.03 > solvers.out.114
-mpirun -np 8 ./new_ij -n 40 40 40 -P 2 2 2 -hmis -interptype 6 -Pmx 4 -nongalerk_tol 3 0.0 0.01 0.05 > solvers.out.115
+mpirun -np 8 ./ij -n 40 40 40 -P 2 2 2 -nongalerk_tol 1 0.03 > solvers.out.114
+mpirun -np 8 ./ij -n 40 40 40 -P 2 2 2 -nongalerk_tol 3 0.0 0.01 0.05 > solvers.out.115
 
 #RAP options
-mpirun -np 8 ./new_ij -n 40 40 40 -P 2 2 2 -pmis -solver 3 -rap 0 > solvers.out.116
-mpirun -np 8 ./new_ij -n 40 40 40 -P 2 2 2 -pmis -solver 3 -rap 1 > solvers.out.117
+mpirun -np 8 ./ij -n 40 40 40 -P 2 2 2 -solver 3 -rap 0 > solvers.out.116
+mpirun -np 8 ./ij -n 40 40 40 -P 2 2 2 -solver 3 -rap 1 > solvers.out.117
diff --git a/src/test/TEST_ij/solvers.saved b/src/test/TEST_ij/solvers.saved
index 93223c3..192947e 100644
--- a/src/test/TEST_ij/solvers.saved
+++ b/src/test/TEST_ij/solvers.saved
@@ -1,22 +1,22 @@
 # Output file: solvers.out.0
-Iterations = 5
-Final Relative Residual Norm = 6.725559e-10
+Iterations = 7
+Final Relative Residual Norm = 3.095059e-09
 
 # Output file: solvers.out.1
 Iterations = 41
 Final Relative Residual Norm = 6.698760e-09
 
 # Output file: solvers.out.2
-GMRES Iterations = 4
-Final GMRES Relative Residual Norm = 7.746171e-09
+GMRES Iterations = 7
+Final GMRES Relative Residual Norm = 4.842561e-09
 
 # Output file: solvers.out.3
 GMRES Iterations = 93
 Final GMRES Relative Residual Norm = 8.225661e-09
 
 # Output file: solvers.out.4
-Iterations = 7
-Final Relative Residual Norm = 2.924249e-09
+Iterations = 9
+Final Relative Residual Norm = 3.723663e-09
 
 # Output file: solvers.out.5
 Iterations = 196
@@ -37,175 +37,75 @@ DSCG_Iterations = 41
 Final Relative Residual Norm = 6.698760e-09
 
 # Output file: solvers.out.9
-Iterations = 8
-PCG_Iterations = 4
+Iterations = 12
+PCG_Iterations = 8
 DSCG_Iterations = 4
-Final Relative Residual Norm = 1.463687e-09
+Final Relative Residual Norm = 2.111630e-09
 
 # Output file: solvers.out.10
-Iterations = 6
-PCG_Iterations = 4
+Iterations = 10
+PCG_Iterations = 8
 DSCG_Iterations = 2
-Final Relative Residual Norm = 7.746171e-09
+Final Relative Residual Norm = 6.719344e-09
 
 # Output file: solvers.out.11
-Iterations = 5
-PCG_Iterations = 2
+Iterations = 7
+PCG_Iterations = 4
 DSCG_Iterations = 3
-Final Relative Residual Norm = 3.025481e-09
-
-# Output file: solvers.out.1.lobpcg
-Iterations = 4
-Final Relative Residual Norm = 6.069126e-08
-
-# Output file: solvers.out.1.lobpcg.1
-Eigenvalue lambda   2.43042158313105e-01
-Residual   4.05590545015100e-07
-
-# Output file: solvers.out.1.lobpcg.5
-Eigenvalue lambda   2.43042158313016e-01
-Eigenvalue lambda   4.79521039879677e-01
-Eigenvalue lambda   4.79521039879696e-01
-Eigenvalue lambda   4.79521039879801e-01
-Eigenvalue lambda   7.15999921446955e-01
-Residual   1.47262238767519e-07
-Residual   4.10326699242317e-07
-Residual   3.64072899758183e-07
-Residual   5.54586913659908e-07
-Residual   1.01332963441651e-06
-
-# Output file: solvers.out.2.lobpcg
-Iterations = 20
-Final Relative Residual Norm = 5.962015e-07
-
-# Output file: solvers.out.2.lobpcg.1
-Eigenvalue lambda   2.43042158313121e-01
-Residual   4.63315030208772e-07
-
-# Output file: solvers.out.2.lobpcg.5
-Eigenvalue lambda   2.43042158313016e-01
-Eigenvalue lambda   4.79521039879677e-01
-Eigenvalue lambda   4.79521039879700e-01
-Eigenvalue lambda   4.79521039879780e-01
-Eigenvalue lambda   7.15999921447091e-01
-Residual   1.43322942857248e-07
-Residual   4.86501096019624e-07
-Residual   3.72232026886917e-07
-Residual   5.18126375223749e-07
-Residual   1.02530795888815e-06
-
-# Output file: solvers.out.8.lobpcg
-Iterations = 15
-Final Relative Residual Norm = 6.352784e-07
-
-# Output file: solvers.out.8.lobpcg.1
-Eigenvalue lambda   2.43042158313109e-01
-Residual   4.35799317942064e-07
-
-# Output file: solvers.out.8.lobpcg.5
-Eigenvalue lambda   2.43042158313016e-01
-Eigenvalue lambda   4.79521039879678e-01
-Eigenvalue lambda   4.79521039879699e-01
-Eigenvalue lambda   4.79521039879799e-01
-Eigenvalue lambda   7.15999921446891e-01
-Residual   1.37749965131754e-07
-Residual   4.63010776375628e-07
-Residual   3.87579831562450e-07
-Residual   5.41642190641787e-07
-Residual   9.41630860798197e-07
-
-# Output file: solvers.out.12.lobpcg
-Iterations = 13
-Final Relative Residual Norm = 3.896456e-07
-
-# Output file: solvers.out.12.lobpcg.1
-Eigenvalue lambda   2.43042158313099e-01
-Residual   3.94665203904632e-07
-
-# Output file: solvers.out.12.lobpcg.5
-Eigenvalue lambda   2.43042158313016e-01
-Eigenvalue lambda   4.79521039879675e-01
-Eigenvalue lambda   4.79521039879685e-01
-Eigenvalue lambda   4.79521039879788e-01
-Eigenvalue lambda   7.15999921447147e-01
-Residual   1.36414438599642e-07
-Residual   4.59913522980684e-07
-Residual   3.72152569537697e-07
-Residual   5.07587948203054e-07
-Residual   1.11208056285383e-06
-
-# Output file: solvers.out.43.lobpcg
-Iterations = 9
-Final Relative Residual Norm = 5.971754e-07
-
-# Output file: solvers.out.43.lobpcg.1
-Eigenvalue lambda   2.43042158313103e-01
-Residual   4.06686999057609e-07
-
-# Output file: solvers.out.43.lobpcg.5
-Eigenvalue lambda   2.43042158313016e-01
-Eigenvalue lambda   4.79521039879682e-01
-Eigenvalue lambda   4.79521039879706e-01
-Eigenvalue lambda   4.79521039879825e-01
-Eigenvalue lambda   7.15999921446956e-01
-Residual   1.61376258673867e-07
-Residual   4.76948835732272e-07
-Residual   4.01718127440120e-07
-Residual   5.82137557078099e-07
-Residual   9.97061314149104e-07
+Final Relative Residual Norm = 6.859548e-09
 
 # Output file: solvers.out.sysh
- Average Convergence Factor = 0.059838
+ Average Convergence Factor = 0.122031
 
-     Complexity:    grid = 1.675375
-                operator = 3.597929
-                   cycle = 7.195690
+     Complexity:    grid = 1.613750
+                operator = 2.860187
+                   cycle = 5.720205
 
 # Output file: solvers.out.sysn
- Average Convergence Factor = 0.044738
+ Average Convergence Factor = 0.232267
 
-     Complexity:    grid = 1.676125
-                operator = 3.605951
-                   cycle = 13.211735
+     Complexity:    grid = 1.592000
+                operator = 2.633619
+                   cycle = 11.267164
 
 # Output file: solvers.out.sysu
- Average Convergence Factor = 0.116978
+ Average Convergence Factor = 0.419936
 
-     Complexity:    grid = 1.680438
-                operator = 3.672533
-                   cycle = 7.344949
+     Complexity:    grid = 1.614937
+                operator = 2.866563
+                   cycle = 5.732747
 
 # Output file: solvers.out.101
 LGMRES Iterations = 83
 Final LGMRES Relative Residual Norm = 8.591967e-09
 
 # Output file: solvers.out.102
-LGMRES Iterations = 4
-Final LGMRES Relative Residual Norm = 7.746171e-09
+LGMRES Iterations = 7
+Final LGMRES Relative Residual Norm = 4.842561e-09
 
 # Output file: solvers.out.103
 FlexGMRES Iterations = 93
 Final FlexGMRES Relative Residual Norm = 8.225661e-09
 
 # Output file: solvers.out.104
-FlexGMRES Iterations = 4
-Final FlexGMRES Relative Residual Norm = 7.746171e-09
+FlexGMRES Iterations = 7
+Final FlexGMRES Relative Residual Norm = 4.842561e-09
 
 # Output file: solvers.out.105
 Iterations = 15
-Final Relative Residual Norm = 3.906540e-09
+Final Relative Residual Norm = 3.104258e-09
 
 # Output file: solvers.out.106
 Iterations = 15
-Final Relative Residual Norm = 3.906540e-09
+Final Relative Residual Norm = 3.104258e-09
 
 # Output file: solvers.out.107
 Iterations = 22
-Final Relative Residual Norm = 4.557728e-09
+Final Relative Residual Norm = 4.329945e-09
 
 # Output file: solvers.out.108
 Iterations = 22
-Final Relative Residual Norm = 4.557728e-09
+Final Relative Residual Norm = 4.329945e-09
 
 # Output file: solvers.out.109
 Iterations = 17
@@ -221,25 +121,25 @@ Final Relative Residual Norm = 8.452076e-09
 
 # Output file: solvers.out.112
 GMRES Iterations = 23
-Final GMRES Relative Residual Norm = 3.943470e-09
+Final GMRES Relative Residual Norm = 3.979845e-09
 
 # Output file: solvers.out.113
 GMRES Iterations = 25
 Final GMRES Relative Residual Norm = 8.744056e-09
 
 # Output file: solvers.out.114
-BoomerAMG Iterations = 13
-Final Relative Residual Norm = 8.867318e-09
+BoomerAMG Iterations = 16
+Final Relative Residual Norm = 9.161445e-09
 
 # Output file: solvers.out.115
-BoomerAMG Iterations = 14
-Final Relative Residual Norm = 3.249548e-09
+BoomerAMG Iterations = 17
+Final Relative Residual Norm = 5.102706e-09
 
 # Output file: solvers.out.116
-GMRES Iterations = 19
-Final GMRES Relative Residual Norm = 3.588416e-09
+GMRES Iterations = 9
+Final GMRES Relative Residual Norm = 9.500151e-09
 
 # Output file: solvers.out.117
-GMRES Iterations = 19
-Final GMRES Relative Residual Norm = 3.341112e-09
+GMRES Iterations = 10
+Final GMRES Relative Residual Norm = 1.006494e-09
 
diff --git a/src/test/TEST_ij/solvers.sh b/src/test/TEST_ij/solvers.sh
index 8ffd8af..c9ac7ec 100755
--- a/src/test/TEST_ij/solvers.sh
+++ b/src/test/TEST_ij/solvers.sh
@@ -12,7 +12,14 @@
 #EHEADER**********************************************************************
 
 TNAME=`basename $0 .sh`
+CONVTOL=$1
 
+# Set default check tolerance
+if [ x$CONVTOL = "x" ];
+then
+    CONVTOL=0.0
+fi
+#echo "tol = $CONVTOL"
 #=============================================================================
 # IJ: Run multiplicative and mult_additive cycle and compare results 
 #                    should be the same
@@ -60,24 +67,6 @@ do
 done >> ${TNAME}.out
 
 FILES="\
- ${TNAME}.out.1.lobpcg\
- ${TNAME}.out.2.lobpcg\
- ${TNAME}.out.8.lobpcg\
- ${TNAME}.out.12.lobpcg\
- ${TNAME}.out.43.lobpcg\
-"
-
-for i in $FILES
-do
-  echo "# Output file: $i"
-  tail -3 $i
-  echo "# Output file: $i.1"
-  tail -13 $i.1 | head -3
-  echo "# Output file: $i.5"
-  tail -21 $i.5 | head -11
-done >> ${TNAME}.out
-
-FILES="\
  ${TNAME}.out.sysh\
  ${TNAME}.out.sysn\
  ${TNAME}.out.sysu\
@@ -124,7 +113,7 @@ if [ "$OUT_COUNT" != "$SAVED_COUNT" ]; then
 fi
 
 if [ -z $HYPRE_NO_SAVED ]; then
-   diff -U3 -bI"time" ${TNAME}.saved ${TNAME}.out >&2
+   (../runcheck.sh ${TNAME}.out ${TNAME}.saved $CONVTOL) >&2
 fi
 
 #=============================================================================
diff --git a/src/test/TEST_sstruct/addtovalues.saved b/src/test/TEST_sstruct/addtovalues.saved
index 00c9c35..b126689 100644
--- a/src/test/TEST_sstruct/addtovalues.saved
+++ b/src/test/TEST_sstruct/addtovalues.saved
@@ -7,12 +7,12 @@ Iterations = 9
 Final Relative Residual Norm = 7.226958e-07
 
 # Output file: addtovalues.out.2
-Iterations = 100
-Final Relative Residual Norm = 4.017242e-05
+Iterations = 34
+Final Relative Residual Norm = 5.437407e-07
 
 # Output file: addtovalues.out.3
-Iterations = 100
-Final Relative Residual Norm = 4.017242e-05
+Iterations = 34
+Final Relative Residual Norm = 5.437407e-07
 
 # Output file: addtovalues.out.4
 Iterations = 11
@@ -23,10 +23,10 @@ Iterations = 11
 Final Relative Residual Norm = 9.051371e-07
 
 # Output file: addtovalues.out.6
-Iterations = 100
-Final Relative Residual Norm = 4.024926e-05
+Iterations = 44
+Final Relative Residual Norm = 8.778287e-07
 
 # Output file: addtovalues.out.7
-Iterations = 100
-Final Relative Residual Norm = 4.024926e-05
+Iterations = 44
+Final Relative Residual Norm = 8.778287e-07
 
diff --git a/src/test/TEST_sstruct/amr2d.saved b/src/test/TEST_sstruct/amr2d.saved
index 15c17e1..42a57fa 100644
--- a/src/test/TEST_sstruct/amr2d.saved
+++ b/src/test/TEST_sstruct/amr2d.saved
@@ -7,12 +7,12 @@ Iterations = 19
 Final Relative Residual Norm = 8.188904e-07
 
 # Output file: amr2d.out.2
-Iterations = 3
-Final Relative Residual Norm = 5.260051e-08
+Iterations = 5
+Final Relative Residual Norm = 1.746768e-07
 
 # Output file: amr2d.out.3
-Iterations = 3
-Final Relative Residual Norm = 5.496176e-08
+Iterations = 5
+Final Relative Residual Norm = 3.155589e-07
 
 # Output file: amr2d.out.4
 Iterations = 20
@@ -23,24 +23,24 @@ Iterations = 20
 Final Relative Residual Norm = 8.510163e-07
 
 # Output file: amr2d.out.6
-Iterations = 3
-Final Relative Residual Norm = 6.033671e-08
+Iterations = 5
+Final Relative Residual Norm = 1.646222e-07
 
 # Output file: amr2d.out.7
-Iterations = 3
-Final Relative Residual Norm = 7.452823e-08
+Iterations = 5
+Final Relative Residual Norm = 4.675292e-07
 
 # Output file: amr2d.out.8
 Iterations = 21
 Final Relative Residual Norm = 5.392338e-07
 
 # Output file: amr2d.out.10
-Iterations = 3
-Final Relative Residual Norm = 5.312236e-08
+Iterations = 6
+Final Relative Residual Norm = 6.135506e-07
 
 # Output file: amr2d.out.11
-Iterations = 3
-Final Relative Residual Norm = 7.814210e-08
+Iterations = 6
+Final Relative Residual Norm = 1.744213e-07
 
 # Output file: amr2d.out.12
 Iterations = 18
diff --git a/src/test/TEST_sstruct/emptyProc.saved b/src/test/TEST_sstruct/emptyProc.saved
index 857911d..aa6ba9d 100644
--- a/src/test/TEST_sstruct/emptyProc.saved
+++ b/src/test/TEST_sstruct/emptyProc.saved
@@ -19,8 +19,8 @@ Iterations = 20
 Final Relative Residual Norm = 5.962015e-07
 
 # Output file: emptyProc.out.05
-Iterations = 4
-Final Relative Residual Norm = 1.317418e-07
+Iterations = 6
+Final Relative Residual Norm = 2.350489e-07
 
 # Output file: emptyProc.out.06
 Iterations = 15
@@ -43,8 +43,8 @@ Iterations = 64
 Final Relative Residual Norm = 9.962554e-07
 
 # Output file: emptyProc.out.12
-Iterations = 4
-Final Relative Residual Norm = 5.827217e-08
+Iterations = 6
+Final Relative Residual Norm = 3.354751e-07
 
 # Output file: emptyProc.out.14
 Iterations = 17
@@ -63,8 +63,8 @@ Iterations = 14
 Final Relative Residual Norm = 6.421347e-07
 
 # Output file: emptyProc.out.18
-Iterations = 2
-Final Relative Residual Norm = 6.547049e-08
+Iterations = 3
+Final Relative Residual Norm = 8.029310e-07
 
 # Output file: emptyProc.out.20
 Iterations = 8
diff --git a/src/test/TEST_sstruct/maxwell.saved b/src/test/TEST_sstruct/maxwell.saved
index 03be30b..1631b50 100644
--- a/src/test/TEST_sstruct/maxwell.saved
+++ b/src/test/TEST_sstruct/maxwell.saved
@@ -1,12 +1,12 @@
 # Output file: maxwell.out.0
 Iterations = 10
-Final Relative Residual Norm = 2.476399e-09
+Final Relative Residual Norm = 2.477002e-09
 
 # Output file: maxwell.out.1
 Iterations = 10
-Final Relative Residual Norm = 3.771640e-09
+Final Relative Residual Norm = 3.772486e-09
 
 # Output file: maxwell.out.2
 Iterations = 10
-Final Relative Residual Norm = 7.153795e-09
+Final Relative Residual Norm = 7.155830e-09
 
diff --git a/src/test/TEST_sstruct/miller.jobs b/src/test/TEST_sstruct/miller.jobs
index ebf90db..5c540c7 100755
--- a/src/test/TEST_sstruct/miller.jobs
+++ b/src/test/TEST_sstruct/miller.jobs
@@ -24,7 +24,7 @@
 mpirun -np 2 ./sstruct -in sstruct.in.miller_2blk  -r 5 10 1 -P 2 1 1 \
  -solver 11 > miller.out.0
 mpirun -np 2 ./sstruct -in sstruct.in.miller_2blk  -r 5 10 1 -P 2 1 1 \
- -solver 20 > miller.out.1
+ -solver 20 -old_default > miller.out.1
 mpirun -np 2 ./sstruct -in sstruct.in.miller_2blk1 -r 5 10 1 -P 2 1 1 \
  -solver 11 > miller.out.2
 mpirun -np 2 ./sstruct -in sstruct.in.miller_2blk1 -r 5 10 1 -P 2 1 1 \
@@ -32,11 +32,11 @@ mpirun -np 2 ./sstruct -in sstruct.in.miller_2blk1 -r 5 10 1 -P 2 1 1 \
 mpirun -np 2 ./sstruct -in sstruct.in.miller_2stk  -r 5 10 1 -P 2 1 1 \
  -solver 11 > miller.out.4
 mpirun -np 2 ./sstruct -in sstruct.in.miller_2stk  -r 5 10 1 -P 2 1 1 \
- -solver 20 > miller.out.5
+ -solver 20 -old_default > miller.out.5
 mpirun -np 2 ./sstruct -in sstruct.in.miller_3blk  -r 5 10 1 -P 2 1 1 \
  -solver 11 > miller.out.6
 mpirun -np 2 ./sstruct -in sstruct.in.miller_3blk  -r 5 10 1 -P 2 1 1 \
- -solver 20 > miller.out.7
+ -solver 20 -old_default > miller.out.7
 mpirun -np 2 ./sstruct -in sstruct.in.miller_diag  -r 10 20 1 -P 2 1 1 \
  -solver 11 > miller.out.8
 mpirun -np 2 ./sstruct -in sstruct.in.miller_diag  -r 10 20 1 -P 2 1 1 \
diff --git a/src/test/TEST_sstruct/miller.saved b/src/test/TEST_sstruct/miller.saved
index e13b022..1acfb9a 100644
--- a/src/test/TEST_sstruct/miller.saved
+++ b/src/test/TEST_sstruct/miller.saved
@@ -3,38 +3,38 @@ Iterations = 16
 Final Relative Residual Norm = 8.455333e-07
 
 # Output file: miller.out.1
-Iterations = 4
-Final Relative Residual Norm = 2.469783e-07
+Iterations = 6
+Final Relative Residual Norm = 9.625406e-08
 
 # Output file: miller.out.2
 Iterations = 15
 Final Relative Residual Norm = 6.496892e-07
 
 # Output file: miller.out.3
-Iterations = 6
-Final Relative Residual Norm = 3.525486e-07
+Iterations = 5
+Final Relative Residual Norm = 4.230245e-07
 
 # Output file: miller.out.4
 Iterations = 18
 Final Relative Residual Norm = 3.006677e-07
 
 # Output file: miller.out.5
-Iterations = 4
-Final Relative Residual Norm = 3.976133e-07
+Iterations = 6
+Final Relative Residual Norm = 8.758757e-08
 
 # Output file: miller.out.6
 Iterations = 37
 Final Relative Residual Norm = 6.181674e-07
 
 # Output file: miller.out.7
-Iterations = 6
-Final Relative Residual Norm = 2.158264e-07
+Iterations = 5
+Final Relative Residual Norm = 8.323418e-07
 
 # Output file: miller.out.8
 Iterations = 3
 Final Relative Residual Norm = 4.822192e-07
 
 # Output file: miller.out.9
-Iterations = 3
-Final Relative Residual Norm = 1.634635e-08
+Iterations = 2
+Final Relative Residual Norm = 2.571870e-07
 
diff --git a/src/test/TEST_sstruct/neumann.jobs b/src/test/TEST_sstruct/neumann.jobs
index b744ca0..5a8810c 100755
--- a/src/test/TEST_sstruct/neumann.jobs
+++ b/src/test/TEST_sstruct/neumann.jobs
@@ -11,26 +11,34 @@
 # $Revision$
 #EHEADER**********************************************************************
 
-
-
-
-
-
-
 #=============================================================================
 # sstruct: Run base 3d cases
 #=============================================================================
 
-mpirun -np 1 ./sstruct -in sstruct.in.neumann -r 6 6 6 -solver 200 \
- > neumann.out.0
-mpirun -np 1 ./sstruct -in sstruct.in.neumann -r 6 6 6 -solver 201 \
- > neumann.out.1
+# Neumann in x, y, and z
+mpirun -np 1 ./sstruct -in sstruct.in.neumann -r 6 6 6 -solver 200     > neumann.out.0
+mpirun -np 1 ./sstruct -in sstruct.in.neumann -r 6 6 6 -solver 201     > neumann.out.1
+
+# Neumann in y and z (Dirichlet in x)
+mpirun -np 1 ./sstruct -in sstruct.in.neumann-yz -r 6 6 6 -solver 200  > neumann.out.10
+mpirun -np 1 ./sstruct -in sstruct.in.neumann-yz -r 6 6 6 -solver 201  > neumann.out.11
+
+# All Dirichlet just for fun
+mpirun -np 1 ./sstruct -in sstruct.in.dirichlet -r 6 6 6 -solver 200   > neumann.out.20
+mpirun -np 1 ./sstruct -in sstruct.in.dirichlet -r 6 6 6 -solver 201   > neumann.out.21
 
 #=============================================================================
 # sstruct: Run various blockings and distributions
 #=============================================================================
 
-mpirun -np 4 ./sstruct -in sstruct.in.neumann -P 2 2 1 -b 1 3 2 -r 3 1 3 -solver 200 \
- > neumann.out.2
-mpirun -np 3 ./sstruct -in sstruct.in.neumann -P 1 1 3 -b 6 1 1 -r 1 6 2 -solver 201 \
- > neumann.out.3
+# Neumann in x, y, and z
+mpirun -np 4 ./sstruct -in sstruct.in.neumann -P 2 2 1 -b 1 3 2 -r 3 1 3 -solver 200     > neumann.out.2
+mpirun -np 3 ./sstruct -in sstruct.in.neumann -P 1 1 3 -b 6 1 1 -r 1 6 2 -solver 201     > neumann.out.3
+
+# Neumann in y and z (Dirichlet in x)
+mpirun -np 4 ./sstruct -in sstruct.in.neumann-yz -P 2 2 1 -b 1 3 2 -r 3 1 3 -solver 200  > neumann.out.12
+mpirun -np 3 ./sstruct -in sstruct.in.neumann-yz -P 1 1 3 -b 6 1 1 -r 1 6 2 -solver 201  > neumann.out.13
+
+# All Dirichlet just for fun
+mpirun -np 4 ./sstruct -in sstruct.in.dirichlet -P 2 2 1 -b 1 3 2 -r 3 1 3 -solver 200   > neumann.out.22
+mpirun -np 3 ./sstruct -in sstruct.in.dirichlet -P 1 1 3 -b 6 1 1 -r 1 6 2 -solver 201   > neumann.out.23
diff --git a/src/test/TEST_sstruct/neumann.saved b/src/test/TEST_sstruct/neumann.saved
index f5bb471..1839e75 100644
--- a/src/test/TEST_sstruct/neumann.saved
+++ b/src/test/TEST_sstruct/neumann.saved
@@ -14,3 +14,35 @@ Final Relative Residual Norm = 2.366245e-07
 Iterations = 22
 Final Relative Residual Norm = 8.557340e-07
 
+# Output file: neumann.out.10
+Iterations = 6
+Final Relative Residual Norm = 1.060090e-07
+
+# Output file: neumann.out.11
+Iterations = 17
+Final Relative Residual Norm = 9.831767e-07
+
+# Output file: neumann.out.12
+Iterations = 6
+Final Relative Residual Norm = 1.060090e-07
+
+# Output file: neumann.out.13
+Iterations = 17
+Final Relative Residual Norm = 9.831767e-07
+
+# Output file: neumann.out.20
+Iterations = 4
+Final Relative Residual Norm = 7.968831e-07
+
+# Output file: neumann.out.21
+Iterations = 16
+Final Relative Residual Norm = 7.270873e-07
+
+# Output file: neumann.out.22
+Iterations = 4
+Final Relative Residual Norm = 7.968831e-07
+
+# Output file: neumann.out.23
+Iterations = 16
+Final Relative Residual Norm = 7.270873e-07
+
diff --git a/src/test/TEST_sstruct/neumann.sh b/src/test/TEST_sstruct/neumann.sh
index 8fbb210..801a7a2 100755
--- a/src/test/TEST_sstruct/neumann.sh
+++ b/src/test/TEST_sstruct/neumann.sh
@@ -11,10 +11,6 @@
 # $Revision$
 #EHEADER**********************************************************************
 
-
-
-
-
 TNAME=`basename $0 .sh`
 
 #=============================================================================
@@ -25,12 +21,28 @@ tail -3 ${TNAME}.out.0 > ${TNAME}.testdata
 tail -3 ${TNAME}.out.2 > ${TNAME}.testdata.temp
 diff ${TNAME}.testdata ${TNAME}.testdata.temp >&2
 
+tail -3 ${TNAME}.out.10 > ${TNAME}.testdata
+tail -3 ${TNAME}.out.12 > ${TNAME}.testdata.temp
+diff ${TNAME}.testdata ${TNAME}.testdata.temp >&2
+
+tail -3 ${TNAME}.out.20 > ${TNAME}.testdata
+tail -3 ${TNAME}.out.22 > ${TNAME}.testdata.temp
+diff ${TNAME}.testdata ${TNAME}.testdata.temp >&2
+
 #=============================================================================
 
 tail -3 ${TNAME}.out.1 > ${TNAME}.testdata
 tail -3 ${TNAME}.out.3 > ${TNAME}.testdata.temp
 diff ${TNAME}.testdata ${TNAME}.testdata.temp >&2
 
+tail -3 ${TNAME}.out.11 > ${TNAME}.testdata
+tail -3 ${TNAME}.out.13 > ${TNAME}.testdata.temp
+diff ${TNAME}.testdata ${TNAME}.testdata.temp >&2
+
+tail -3 ${TNAME}.out.21 > ${TNAME}.testdata
+tail -3 ${TNAME}.out.23 > ${TNAME}.testdata.temp
+diff ${TNAME}.testdata ${TNAME}.testdata.temp >&2
+
 #=============================================================================
 # compare with baseline case
 #=============================================================================
@@ -40,6 +52,14 @@ FILES="\
  ${TNAME}.out.1\
  ${TNAME}.out.2\
  ${TNAME}.out.3\
+ ${TNAME}.out.10\
+ ${TNAME}.out.11\
+ ${TNAME}.out.12\
+ ${TNAME}.out.13\
+ ${TNAME}.out.20\
+ ${TNAME}.out.21\
+ ${TNAME}.out.22\
+ ${TNAME}.out.23\
 "
 
 for i in $FILES
diff --git a/src/test/TEST_sstruct/sharedpart.saved b/src/test/TEST_sstruct/sharedpart.saved
index 0da5b0c..6228064 100644
--- a/src/test/TEST_sstruct/sharedpart.saved
+++ b/src/test/TEST_sstruct/sharedpart.saved
@@ -23,18 +23,18 @@ Iterations = 18
 Final Relative Residual Norm = 4.922026e-07
 
 # Output file: sharedpart.out.50
-Iterations = 3
-Final Relative Residual Norm = 3.817579e-08
+Iterations = 5
+Final Relative Residual Norm = 1.368642e-07
 
 # Output file: sharedpart.out.51
-Iterations = 3
-Final Relative Residual Norm = 4.083987e-08
+Iterations = 5
+Final Relative Residual Norm = 1.549669e-07
 
 # Output file: sharedpart.out.52
-Iterations = 3
-Final Relative Residual Norm = 3.817579e-08
+Iterations = 5
+Final Relative Residual Norm = 1.368642e-07
 
 # Output file: sharedpart.out.53
 Iterations = 7
-Final Relative Residual Norm = 3.576822e-07
+Final Relative Residual Norm = 2.557686e-07
 
diff --git a/src/test/TEST_sstruct/solvers.saved b/src/test/TEST_sstruct/solvers.saved
index cee45a4..63f68a8 100644
--- a/src/test/TEST_sstruct/solvers.saved
+++ b/src/test/TEST_sstruct/solvers.saved
@@ -16,7 +16,7 @@ Final Relative Residual Norm = 9.124482e-07
 
 # Output file: solvers.out.4
 Iterations = 4
-Final Relative Residual Norm = 5.458858e-08
+Final Relative Residual Norm = 9.533464e-07
 
 # Output file: solvers.out.5
 Iterations = 7
@@ -44,7 +44,7 @@ Final Relative Residual Norm = 9.036190e-07
 
 # Output file: solvers.out.11
 Iterations = 4
-Final Relative Residual Norm = 3.500192e-08
+Final Relative Residual Norm = 9.340817e-07
 
 # Output file: solvers.out.12
 Iterations = 7
@@ -60,7 +60,7 @@ Final Relative Residual Norm = 8.131824e-07
 
 # Output file: solvers.out.15
 Iterations = 5
-Final Relative Residual Norm = 9.410367e-07
+Final Relative Residual Norm = 4.012141e-07
 
 # Output file: solvers.out.16
 Iterations = 28
@@ -76,11 +76,11 @@ Final Relative Residual Norm = 9.036190e-07
 
 # Output file: solvers.out.19
 Iterations = 4
-Final Relative Residual Norm = 3.500192e-08
+Final Relative Residual Norm = 9.340817e-07
 
 # Output file: solvers.out.20
 Iterations = 4
-Final Relative Residual Norm = 3.500192e-08
+Final Relative Residual Norm = 9.340817e-07
 
 # Output file: solvers.out.10.lobpcg
 Iterations = 24
diff --git a/src/test/TEST_sstruct/solvers.sh b/src/test/TEST_sstruct/solvers.sh
index d7e9cfb..d15cc7e 100755
--- a/src/test/TEST_sstruct/solvers.sh
+++ b/src/test/TEST_sstruct/solvers.sh
@@ -16,7 +16,14 @@
 
 
 TNAME=`basename $0 .sh`
+CONVTOL=$1
 
+# Set default check tolerance
+if [ x$CONVTOL = "x" ];
+then
+    CONVTOL=0.0
+fi
+#echo "tol = $CONVTOL"
 #=============================================================================
 # compare with baseline case
 #=============================================================================
@@ -77,7 +84,7 @@ if [ "$OUT_COUNT" != "$SAVED_COUNT" ]; then
 fi
 
 if [ -z $HYPRE_NO_SAVED ]; then
-   diff -U3 -bI"time" ${TNAME}.saved ${TNAME}.out >&2
+   (../runcheck.sh ${TNAME}.out ${TNAME}.saved $CONVTOL) >&2
 fi
 
 #=============================================================================
diff --git a/src/test/TEST_sstruct/sstruct.in.dirichlet b/src/test/TEST_sstruct/sstruct.in.dirichlet
new file mode 100644
index 0000000..92d8933
--- /dev/null
+++ b/src/test/TEST_sstruct/sstruct.in.dirichlet
@@ -0,0 +1,65 @@
+
+###########################################################
+
+# GridCreate: ndim nparts
+GridCreate: 3 1
+
+# GridSetExtents: part ilower(ndim) iupper(ndim)
+GridSetExtents: 0 (1- 1- 1-) (2+ 2+ 2+)
+
+# GridSetVariables: part nvars vartypes[nvars]
+# CELL  = 0
+GridSetVariables: 0 1 [0]
+
+###########################################################
+
+# StencilCreate: nstencils sizes[nstencils]
+StencilCreate: 1 [7]
+
+# StencilSetEntry: stencil_num entry offset[ndim] var value
+StencilSetEntry: 0  0 [ 0  0  0] 0  6.0
+StencilSetEntry: 0  1 [-1  0  0] 0 -1.0
+StencilSetEntry: 0  2 [ 1  0  0] 0 -1.0
+StencilSetEntry: 0  3 [ 0 -1  0] 0 -1.0
+StencilSetEntry: 0  4 [ 0  1  0] 0 -1.0
+StencilSetEntry: 0  5 [ 0  0 -1] 0 -1.0
+StencilSetEntry: 0  6 [ 0  0  1] 0 -1.0
+
+###########################################################
+
+# GraphSetStencil: part var stencil_num
+GraphSetStencil: 0 0 0
+
+###########################################################
+
+# MatrixSetValues: \
+#   part ilower(ndim) iupper(ndim) stride[ndim] var entry value
+
+MatrixSetValues: 0 (1- 1- 1-) (1- 2+ 2+) [1 1 1] 0 1 0.0
+# MatrixSetValues: 0 (1- 1- 1-) (1- 2+ 2+) [1 1 1] 0 0 5.0
+
+MatrixSetValues: 0 (2+ 1- 1-) (2+ 2+ 2+) [1 1 1] 0 2 0.0
+# MatrixSetValues: 0 (2+ 1- 1-) (2+ 2+ 2+) [1 1 1] 0 0 5.0
+
+MatrixSetValues: 0 (1- 1- 1-) (2+ 1- 2+) [1 1 1] 0 3 0.0
+# MatrixSetValues: 0 (1- 1- 1-) (2+ 1- 2+) [1 1 1] 0 0 5.0
+
+MatrixSetValues: 0 (1- 2+ 1-) (2+ 2+ 2+) [1 1 1] 0 4 0.0
+# MatrixSetValues: 0 (1- 2+ 1-) (2+ 2+ 2+) [1 1 1] 0 0 5.0
+
+MatrixSetValues: 0 (1- 1- 1-) (2+ 2+ 1-) [1 1 1] 0 5 0.0
+# MatrixSetValues: 0 (1- 1- 1-) (2+ 2+ 1-) [1 1 1] 0 0 5.0
+
+MatrixSetValues: 0 (1- 1- 2+) (2+ 2+ 2+) [1 1 1] 0 6 0.0
+# MatrixSetValues: 0 (1- 1- 2+) (2+ 2+ 2+) [1 1 1] 0 0 5.0
+
+###########################################################
+
+# ProcessPoolCreate: num_pools
+ProcessPoolCreate: 1
+
+# ProcessPoolSetPart: pool part
+ProcessPoolSetPart: 0 0
+
+###########################################################
+
diff --git a/src/test/TEST_sstruct/sstruct.in.neumann-yz b/src/test/TEST_sstruct/sstruct.in.neumann-yz
new file mode 100644
index 0000000..3e511fe
--- /dev/null
+++ b/src/test/TEST_sstruct/sstruct.in.neumann-yz
@@ -0,0 +1,90 @@
+
+###########################################################
+
+# GridCreate: ndim nparts
+GridCreate: 3 1
+
+# GridSetExtents: part ilower(ndim) iupper(ndim)
+GridSetExtents: 0 (1- 1- 1-) (2+ 2+ 2+)
+
+# GridSetVariables: part nvars vartypes[nvars]
+# CELL  = 0
+GridSetVariables: 0 1 [0]
+
+###########################################################
+
+# StencilCreate: nstencils sizes[nstencils]
+StencilCreate: 1 [7]
+
+# StencilSetEntry: stencil_num entry offset[ndim] var value
+StencilSetEntry: 0  0 [ 0  0  0] 0  6.0
+StencilSetEntry: 0  1 [-1  0  0] 0 -1.0
+StencilSetEntry: 0  2 [ 1  0  0] 0 -1.0
+StencilSetEntry: 0  3 [ 0 -1  0] 0 -1.0
+StencilSetEntry: 0  4 [ 0  1  0] 0 -1.0
+StencilSetEntry: 0  5 [ 0  0 -1] 0 -1.0
+StencilSetEntry: 0  6 [ 0  0  1] 0 -1.0
+
+###########################################################
+
+# GraphSetStencil: part var stencil_num
+GraphSetStencil: 0 0 0
+
+###########################################################
+
+# MatrixSetValues: \
+#   part ilower(ndim) iupper(ndim) stride[ndim] var entry value
+
+MatrixSetValues: 0 (1- 1- 1-) (1- 2+ 2+) [1 1 1] 0 1 0.0
+# MatrixSetValues: 0 (1- 1- 1-) (1- 2+ 2+) [1 1 1] 0 0 5.0
+
+MatrixSetValues: 0 (2+ 1- 1-) (2+ 2+ 2+) [1 1 1] 0 2 0.0
+# MatrixSetValues: 0 (2+ 1- 1-) (2+ 2+ 2+) [1 1 1] 0 0 5.0
+
+MatrixSetValues: 0 (1- 1- 1-) (2+ 1- 2+) [1 1 1] 0 3 0.0
+MatrixSetValues: 0 (1- 1- 1-) (2+ 1- 2+) [1 1 1] 0 0 5.0
+
+MatrixSetValues: 0 (1- 2+ 1-) (2+ 2+ 2+) [1 1 1] 0 4 0.0
+MatrixSetValues: 0 (1- 2+ 1-) (2+ 2+ 2+) [1 1 1] 0 0 5.0
+
+MatrixSetValues: 0 (1- 1- 1-) (2+ 2+ 1-) [1 1 1] 0 5 0.0
+MatrixSetValues: 0 (1- 1- 1-) (2+ 2+ 1-) [1 1 1] 0 0 5.0
+
+MatrixSetValues: 0 (1- 1- 2+) (2+ 2+ 2+) [1 1 1] 0 6 0.0
+MatrixSetValues: 0 (1- 1- 2+) (2+ 2+ 2+) [1 1 1] 0 0 5.0
+
+MatrixSetValues: 0 (1- 2+ 2+) (2+ 2+ 2+) [1 1 1] 0 0 4.0
+MatrixSetValues: 0 (2+ 1- 2+) (2+ 2+ 2+) [1 1 1] 0 0 5.0
+MatrixSetValues: 0 (2+ 2+ 1-) (2+ 2+ 2+) [1 1 1] 0 0 5.0
+
+MatrixSetValues: 0 (1- 1- 1-) (1- 1- 2+) [1 1 1] 0 0 5.0
+MatrixSetValues: 0 (1- 1- 2+) (1- 2+ 2+) [1 1 1] 0 0 5.0
+MatrixSetValues: 0 (1- 1- 2+) (2+ 1- 2+) [1 1 1] 0 0 4.0
+
+MatrixSetValues: 0 (1- 2+ 1-) (1- 2+ 2+) [1 1 1] 0 0 5.0
+MatrixSetValues: 0 (1- 1- 1-) (1- 2+ 1-) [1 1 1] 0 0 5.0
+MatrixSetValues: 0 (1- 2+ 1-) (2+ 2+ 1-) [1 1 1] 0 0 4.0
+
+MatrixSetValues: 0 (2+ 1- 1-) (2+ 1- 2+) [1 1 1] 0 0 5.0
+MatrixSetValues: 0 (1- 1- 1-) (2+ 1- 1-) [1 1 1] 0 0 4.0
+MatrixSetValues: 0 (2+ 1- 1-) (2+ 2+ 1-) [1 1 1] 0 0 5.0
+
+MatrixSetValues: 0 (1- 1- 1-) (1- 1- 1-) [1 1 1] 0 0 4.0
+MatrixSetValues: 0 (2+ 1- 1-) (2+ 1- 1-) [1 1 1] 0 0 4.0
+MatrixSetValues: 0 (1- 2+ 1-) (1- 2+ 1-) [1 1 1] 0 0 4.0
+MatrixSetValues: 0 (2+ 2+ 1-) (2+ 2+ 1-) [1 1 1] 0 0 4.0
+MatrixSetValues: 0 (1- 1- 2+) (1- 1- 2+) [1 1 1] 0 0 4.0
+MatrixSetValues: 0 (2+ 1- 2+) (2+ 1- 2+) [1 1 1] 0 0 4.0
+MatrixSetValues: 0 (1- 2+ 2+) (1- 2+ 2+) [1 1 1] 0 0 4.0
+MatrixSetValues: 0 (2+ 2+ 2+) (2+ 2+ 2+) [1 1 1] 0 0 4.0
+
+###########################################################
+
+# ProcessPoolCreate: num_pools
+ProcessPoolCreate: 1
+
+# ProcessPoolSetPart: pool part
+ProcessPoolSetPart: 0 0
+
+###########################################################
+
diff --git a/src/test/TEST_sstruct/sstruct.in.wide.2D b/src/test/TEST_sstruct/sstruct.in.wide.2D
new file mode 100644
index 0000000..6541662
--- /dev/null
+++ b/src/test/TEST_sstruct/sstruct.in.wide.2D
@@ -0,0 +1,68 @@
+
+###########################################################
+
+# GridCreate: ndim nparts
+GridCreate: 2 1
+
+# GridSetNumGhost: numghost[2*ndim]
+GridSetNumGhost: [3 3 3 3]
+
+# GridSetExtents: part ilower(ndim) iupper(ndim)
+GridSetExtents: 0 (1- 1-) (2+ 2+)
+
+# GridSetVariables: part nvars vartypes[nvars]
+# CELL  = 0
+GridSetVariables: 0 1 [0]
+
+###########################################################
+
+# StencilCreate: nstencils sizes[nstencils]
+StencilCreate: 1 [13]
+
+# StencilSetEntry: stencil_num entry offset[ndim] var value
+StencilSetEntry: 0  0 [ 0  0] 0 12.0
+StencilSetEntry: 0  1 [-1  0] 0 -1.0
+StencilSetEntry: 0  2 [ 1  0] 0 -1.0
+StencilSetEntry: 0  3 [ 0 -1] 0 -1.0
+StencilSetEntry: 0  4 [ 0  1] 0 -1.0
+StencilSetEntry: 0  5 [-2  0] 0 -1.0
+StencilSetEntry: 0  6 [ 2  0] 0 -1.0
+StencilSetEntry: 0  7 [ 0 -2] 0 -1.0
+StencilSetEntry: 0  8 [ 0  2] 0 -1.0
+StencilSetEntry: 0  9 [-3  0] 0 -1.0
+StencilSetEntry: 0 10 [ 3  0] 0 -1.0
+StencilSetEntry: 0 11 [ 0 -3] 0 -1.0
+StencilSetEntry: 0 12 [ 0  3] 0 -1.0
+
+###########################################################
+
+# GraphSetStencil: part var stencil_num
+GraphSetStencil: 0 0 0
+
+###########################################################
+
+# MatrixSetValues: \
+#   part ilower(ndim) iupper(ndim) stride[ndim] var entry value
+MatrixSetValues: 0 (1- 1-) (1- 2+) [1 1] 0  1 0.0
+MatrixSetValues: 0 (2+ 1-) (2+ 2+) [1 1] 0  2 0.0
+MatrixSetValues: 0 (1- 1-) (2+ 1-) [1 1] 0  3 0.0
+MatrixSetValues: 0 (1- 2+) (2+ 2+) [1 1] 0  4 0.0
+MatrixSetValues: 0 (1- 1-) (1- 2+) [1 1] 0  5 0.0
+MatrixSetValues: 0 (2+ 1-) (2+ 2+) [1 1] 0  6 0.0
+MatrixSetValues: 0 (1- 1-) (2+ 1-) [1 1] 0  7 0.0
+MatrixSetValues: 0 (1- 2+) (2+ 2+) [1 1] 0  8 0.0
+MatrixSetValues: 0 (1- 1-) (1- 2+) [1 1] 0  9 0.0
+MatrixSetValues: 0 (2+ 1-) (2+ 2+) [1 1] 0 10 0.0
+MatrixSetValues: 0 (1- 1-) (2+ 1-) [1 1] 0 11 0.0
+MatrixSetValues: 0 (1- 2+) (2+ 2+) [1 1] 0 12 0.0
+
+###########################################################
+
+# ProcessPoolCreate: num_pools
+ProcessPoolCreate: 1
+
+# ProcessPoolSetPart: pool part
+ProcessPoolSetPart: 0 0
+
+###########################################################
+
diff --git a/src/test/TEST_ij/lobpcg.jobs b/src/test/TEST_sstruct/wide.jobs
similarity index 60%
copy from src/test/TEST_ij/lobpcg.jobs
copy to src/test/TEST_sstruct/wide.jobs
index d77ff62..86fa774 100755
--- a/src/test/TEST_ij/lobpcg.jobs
+++ b/src/test/TEST_sstruct/wide.jobs
@@ -11,17 +11,6 @@
 # $Revision$
 #EHEADER**********************************************************************
 
+mpirun -np 1 ./sstruct -in sstruct.in.wide.2D -r 10 10 1 -P 1 1 1 -solver 239 > wide.out.1
+mpirun -np 4 ./sstruct -in sstruct.in.wide.2D -r  5  5 1 -P 2 2 1 -solver 239 > wide.out.4
 
-
-
-
-
-
-#=============================================================================
-# ij: LOBPCG cases
-#=============================================================================
-
-mpirun -np 2 ./ij -lobpcg -vrand 2 -seed 1 -tol 1.e-4 -vout 1 > lobpcg.out.vout.1
-
-# use vectors computed by previous run as the initial
-mpirun -np 2 ./ij -lobpcg -vfromfile -seed 1 > lobpcg.out.vfromfile
diff --git a/src/test/TEST_sstruct/wide.saved b/src/test/TEST_sstruct/wide.saved
new file mode 100644
index 0000000..c8d4ab7
--- /dev/null
+++ b/src/test/TEST_sstruct/wide.saved
@@ -0,0 +1,8 @@
+# Output file: wide.out.1
+Iterations = 49
+Final Relative Residual Norm = 9.478906e-07
+
+# Output file: wide.out.4
+Iterations = 49
+Final Relative Residual Norm = 9.478906e-07
+
diff --git a/src/test/TEST_sstruct/neumann.sh b/src/test/TEST_sstruct/wide.sh
similarity index 81%
copy from src/test/TEST_sstruct/neumann.sh
copy to src/test/TEST_sstruct/wide.sh
index 8fbb210..423e3f2 100755
--- a/src/test/TEST_sstruct/neumann.sh
+++ b/src/test/TEST_sstruct/wide.sh
@@ -11,24 +11,14 @@
 # $Revision$
 #EHEADER**********************************************************************
 
-
-
-
-
 TNAME=`basename $0 .sh`
 
 #=============================================================================
-# sstruct: Test various blockings and distributions of default problem
-#=============================================================================
-
-tail -3 ${TNAME}.out.0 > ${TNAME}.testdata
-tail -3 ${TNAME}.out.2 > ${TNAME}.testdata.temp
-diff ${TNAME}.testdata ${TNAME}.testdata.temp >&2
-
+# sstruct: Test wide stencils
 #=============================================================================
 
 tail -3 ${TNAME}.out.1 > ${TNAME}.testdata
-tail -3 ${TNAME}.out.3 > ${TNAME}.testdata.temp
+tail -3 ${TNAME}.out.4 > ${TNAME}.testdata.temp
 diff ${TNAME}.testdata ${TNAME}.testdata.temp >&2
 
 #=============================================================================
@@ -36,10 +26,8 @@ diff ${TNAME}.testdata ${TNAME}.testdata.temp >&2
 #=============================================================================
 
 FILES="\
- ${TNAME}.out.0\
  ${TNAME}.out.1\
- ${TNAME}.out.2\
- ${TNAME}.out.3\
+ ${TNAME}.out.4\
 "
 
 for i in $FILES
diff --git a/src/test/TEST_struct/pfmgvcycle.sh b/src/test/TEST_struct/pfmgvcycle.sh
index 09555ce..3c5c1f3 100755
--- a/src/test/TEST_struct/pfmgvcycle.sh
+++ b/src/test/TEST_struct/pfmgvcycle.sh
@@ -16,7 +16,14 @@
 
 
 TNAME=`basename $0 .sh`
+CONVTOL=$1
 
+# Set default check tolerance
+if [ x$CONVTOL = "x" ]; 
+then
+    CONVTOL=0.0
+fi
+#echo "tol = $CONVTOL"
 #=============================================================================
 # compare with baseline case
 #=============================================================================
@@ -41,7 +48,8 @@ if [ "$OUT_COUNT" != "$SAVED_COUNT" ]; then
 fi
 
 if [ -z $HYPRE_NO_SAVED ]; then
-   diff -U3 -bI"time" ${TNAME}.saved ${TNAME}.out >&2
+#   diff -U3 -bI"time" ${TNAME}.saved ${TNAME}.out >&2
+   (../runcheck.sh ${TNAME}.out ${TNAME}.saved $CONVTOL) >&2
 fi
 
 #=============================================================================
diff --git a/src/test/TEST_struct/solvers.sh b/src/test/TEST_struct/solvers.sh
index bbb6a48..d93c101 100755
--- a/src/test/TEST_struct/solvers.sh
+++ b/src/test/TEST_struct/solvers.sh
@@ -16,7 +16,14 @@
 
 
 TNAME=`basename $0 .sh`
+CONVTOL=$1
 
+# Set default check tolerance
+if [ x$CONVTOL = "x" ];
+then
+    CONVTOL=0.0
+fi
+#echo "tol = $CONVTOL"
 #=============================================================================
 # compare with baseline case
 #=============================================================================
@@ -62,7 +69,7 @@ if [ "$OUT_COUNT" != "$SAVED_COUNT" ]; then
 fi
 
 if [ -z $HYPRE_NO_SAVED ]; then
-   diff -U3 -bI"time" ${TNAME}.saved ${TNAME}.out >&2
+   (../runcheck.sh ${TNAME}.out ${TNAME}.saved $CONVTOL) >&2
 fi
 
 #=============================================================================
diff --git a/src/test/checktest.sh b/src/test/checktest.sh
new file mode 100755
index 0000000..59508dc
--- /dev/null
+++ b/src/test/checktest.sh
@@ -0,0 +1,64 @@
+#!/bin/sh
+#BHEADER**********************************************************************
+# Copyright (c) 2008,  Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+# This file is part of HYPRE.  See file COPYRIGHT for details.
+#
+# HYPRE is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+#
+# $Revision$
+#EHEADER**********************************************************************
+
+# Echo usage information
+case $1 in
+   -h|-help)
+cat <<EOF
+
+   $0 [-h|-help] [{test_dir}]
+
+   where: {test_dir} is the name of some number of runtest directories
+          -h|-help   prints this usage information and exits
+
+   This script checks the error files for the runtest.sh tests in the specified
+   runtest directories.  If no directory is specified, it is assumed that the
+   script is being run from within the hypre 'test' directory, and all of the
+   'TEST_*' directories are checked.
+
+   Example usage: $0 TEST_struct TEST_ij
+
+EOF
+   exit
+   ;;
+esac
+
+RESET=`shopt -p nullglob`  # Save current nullglob setting
+shopt -s nullglob          # Return an empty string for failed wildcard matches 
+if [ "x$1" = "x" ]
+then
+   testdirs=`echo TEST*`   # All TEST directories
+else
+   testdirs=`echo $*`      # Only the specified test directories
+fi
+$RESET                     # Restore nullglob setting
+
+echo ""
+for testdir in $testdirs
+do
+   files=`find $testdir -name '*.err' | sort`
+   if [ -n "$files" ]
+   then
+      for file in $files
+      do
+         SZ=`ls -l $file | awk '{print $5}'`
+         if [ $SZ != 0 ]
+         then
+            echo "FAILED : $file  ($SZ)"
+         else
+            echo "    OK : $file"
+         fi
+      done
+      echo ""
+   fi
+done
diff --git a/src/test/cleantest.sh b/src/test/cleantest.sh
index 58cb90d..f09bd7e 100755
--- a/src/test/cleantest.sh
+++ b/src/test/cleantest.sh
@@ -11,9 +11,6 @@
 # $Revision$
 #EHEADER**********************************************************************
 
-
-
-
 # Echo usage information
 case $1 in
    -h|-help)
@@ -36,22 +33,20 @@ EOF
    ;;
 esac
 
+RESET=`shopt -p nullglob`  # Save current nullglob setting
+shopt -s nullglob          # Return an empty string for failed wildcard matches 
 if [ "x$1" = "x" ]
 then
-   for testdir in TEST*
-   do
-      rm -f $testdir/*err*
-      rm -f $testdir/*out*
-      rm -f $testdir/*log*
-      rm -f $testdir/*.fil
-   done
+   testdirs=`echo TEST*`   # All TEST directories
 else
-   while [ "$*" ]
-   do
-      rm -f $1/*err*
-      rm -f $1/*out*
-      rm -f $1/*log*
-      rm -f $1/*.fil
-      shift
-   done
+   testdirs=`echo $*`      # Only the specified test directories
 fi
+$RESET                     # Restore nullglob setting
+
+for testdir in $testdirs
+do
+   rm -f $testdir/*err*
+   rm -f $testdir/*out*
+   rm -f $testdir/*log*
+   rm -f $testdir/*.fil
+done
diff --git a/src/test/ij.c b/src/test/ij.c
index e8b737d..76de9da 100644
--- a/src/test/ij.c
+++ b/src/test/ij.c
@@ -49,16 +49,43 @@ BuildParIsoLaplacian( HYPRE_Int argc, char** argv, HYPRE_ParCSRMatrix *A_ptr );
 
 /* end lobpcg */
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 HYPRE_Int BuildParFromFile (HYPRE_Int argc , char *argv [], HYPRE_Int arg_index , HYPRE_ParCSRMatrix *A_ptr );
+HYPRE_Int BuildParRhsFromFile (HYPRE_Int argc , char *argv [], HYPRE_Int arg_index , HYPRE_ParVector *b_ptr );
+
 HYPRE_Int BuildParLaplacian (HYPRE_Int argc , char *argv [], HYPRE_Int arg_index , HYPRE_ParCSRMatrix *A_ptr );
+HYPRE_Int BuildParSysLaplacian (HYPRE_Int argc , char *argv [], HYPRE_Int arg_index , HYPRE_ParCSRMatrix *A_ptr );
 HYPRE_Int BuildParDifConv (HYPRE_Int argc , char *argv [], HYPRE_Int arg_index , HYPRE_ParCSRMatrix *A_ptr );
-HYPRE_Int BuildParFromOneFile2(HYPRE_Int argc , char *argv [], HYPRE_Int arg_index , HYPRE_Int num_functions , HYPRE_ParCSRMatrix *A_ptr );
+HYPRE_Int BuildParFromOneFile (HYPRE_Int argc , char *argv [], HYPRE_Int arg_index , HYPRE_Int num_functions , HYPRE_ParCSRMatrix *A_ptr );
 HYPRE_Int BuildFuncsFromFiles (HYPRE_Int argc , char *argv [], HYPRE_Int arg_index , HYPRE_ParCSRMatrix A , HYPRE_Int **dof_func_ptr );
 HYPRE_Int BuildFuncsFromOneFile (HYPRE_Int argc , char *argv [], HYPRE_Int arg_index , HYPRE_ParCSRMatrix A , HYPRE_Int **dof_func_ptr );
-HYPRE_Int BuildRhsParFromOneFile2(HYPRE_Int argc , char *argv [], HYPRE_Int arg_index , HYPRE_Int *partitioning , HYPRE_ParVector *b_ptr );
+HYPRE_Int BuildRhsParFromOneFile (HYPRE_Int argc , char *argv [], HYPRE_Int arg_index , HYPRE_Int *partitioning , HYPRE_ParVector *b_ptr );
 HYPRE_Int BuildParLaplacian9pt (HYPRE_Int argc , char *argv [], HYPRE_Int arg_index , HYPRE_ParCSRMatrix *A_ptr );
 HYPRE_Int BuildParLaplacian27pt (HYPRE_Int argc , char *argv [], HYPRE_Int arg_index , HYPRE_ParCSRMatrix *A_ptr );
-
+HYPRE_Int BuildParRotate7pt (HYPRE_Int argc , char *argv [], HYPRE_Int arg_index , HYPRE_ParCSRMatrix *A_ptr );
+HYPRE_Int BuildParVarDifConv (HYPRE_Int argc , char *argv [], HYPRE_Int arg_index , HYPRE_ParCSRMatrix *A_ptr , HYPRE_ParVector *rhs_ptr );
+HYPRE_ParCSRMatrix GenerateSysLaplacian (MPI_Comm comm, HYPRE_Int nx, HYPRE_Int ny, HYPRE_Int nz, 
+                                         HYPRE_Int P, HYPRE_Int Q, HYPRE_Int R, HYPRE_Int p, HYPRE_Int q, HYPRE_Int r,
+                                         HYPRE_Int num_fun, HYPRE_Real *mtrx, HYPRE_Real *value);
+HYPRE_ParCSRMatrix GenerateSysLaplacianVCoef (MPI_Comm comm, HYPRE_Int nx, HYPRE_Int ny, HYPRE_Int nz, 
+                                              HYPRE_Int P, HYPRE_Int Q, HYPRE_Int R, HYPRE_Int p, HYPRE_Int q, HYPRE_Int r,
+                                              HYPRE_Int num_fun, HYPRE_Real *mtrx, HYPRE_Real *value);
+HYPRE_Int SetSysVcoefValues(HYPRE_Int num_fun, HYPRE_Int nx, HYPRE_Int ny, HYPRE_Int nz, HYPRE_Real vcx, HYPRE_Real vcy, HYPRE_Real vcz, HYPRE_Int mtx_entry, HYPRE_Real *values);
+
+HYPRE_Int BuildParCoordinates (HYPRE_Int argc , char *argv [], HYPRE_Int arg_index , HYPRE_Int *coorddim_ptr , float **coord_ptr );
+                                                                                
+extern HYPRE_Int hypre_FlexGMRESModifyPCAMGExample(void *precond_data, HYPRE_Int iterations, 
+                                                   HYPRE_Real rel_residual_norm);
+
+extern HYPRE_Int hypre_FlexGMRESModifyPCDefault(void *precond_data, HYPRE_Int iteration, 
+                                                HYPRE_Real rel_residual_norm);
+
+#ifdef __cplusplus
+}
+#endif
 #define SECOND_TIME 0
  
 hypre_int
@@ -68,6 +95,10 @@ main( hypre_int argc,
    HYPRE_Int                 arg_index;
    HYPRE_Int                 print_usage;
    HYPRE_Int                 sparsity_known = 0;
+   HYPRE_Int                 add = 0;
+   HYPRE_Int                 off_proc = 0;
+   HYPRE_Int                 chunk = 0;
+   HYPRE_Int                 omp_flag = 0;
    HYPRE_Int                 build_matrix_type;
    HYPRE_Int                 build_matrix_arg_index;
    HYPRE_Int                 build_rhs_type;
@@ -82,14 +113,14 @@ main( hypre_int argc,
    HYPRE_Int                 poutdat;
    HYPRE_Int                 debug_flag;
    HYPRE_Int                 ierr = 0;
-   HYPRE_Int                 i,j,k; 
-   HYPRE_Int                 indx, rest, tms;
+   HYPRE_Int                 i,j; 
    HYPRE_Int                 max_levels = 25;
-   HYPRE_Int                 num_iterations; 
-   HYPRE_Int                 pcg_num_its; 
-   HYPRE_Int                 dscg_num_its; 
-   HYPRE_Int                 pcg_max_its; 
-   HYPRE_Int                 dscg_max_its; 
+   HYPRE_Int                 num_iterations;
+   HYPRE_Int                 pcg_num_its, dscg_num_its;
+   HYPRE_Int                 max_iter = 1000;
+   HYPRE_Int                 mg_max_iter = 100;
+   HYPRE_Int                 nodal = 0;
+   HYPRE_Int                 nodal_diag = 0;
    HYPRE_Real          cf_tol = 0.9;
    HYPRE_Real          norm;
    HYPRE_Real          final_res_norm;
@@ -98,10 +129,12 @@ main( hypre_int argc,
    HYPRE_IJMatrix      ij_A; 
    HYPRE_IJVector      ij_b;
    HYPRE_IJVector      ij_x;
+   HYPRE_IJVector      *ij_rbm;
 
    HYPRE_ParCSRMatrix  parcsr_A;
    HYPRE_ParVector     b;
    HYPRE_ParVector     x;
+   HYPRE_ParVector     *interp_vecs = NULL;
 
    HYPRE_Solver        amg_solver;
    HYPRE_Solver        pcg_solver;
@@ -112,54 +145,99 @@ main( hypre_int argc,
    HYPRE_Int                *row_sizes;
    HYPRE_Int                *diag_sizes;
    HYPRE_Int                *offdiag_sizes;
+   HYPRE_Int                *rows;
    HYPRE_Int                 size;
+   HYPRE_Int                *ncols;
    HYPRE_Int                *col_inds;
+   HYPRE_Int                *dof_func;
+   HYPRE_Int		       num_functions = 1;
+   HYPRE_Int		       num_paths = 1;
+   HYPRE_Int		       agg_num_levels = 0;
+   HYPRE_Int		       ns_coarse = 1;
 
    HYPRE_Int		       time_index;
    MPI_Comm            comm = hypre_MPI_COMM_WORLD;
    HYPRE_Int M, N;
    HYPRE_Int first_local_row, last_local_row, local_num_rows;
    HYPRE_Int first_local_col, last_local_col, local_num_cols;
-   HYPRE_Int local_num_vars;
    HYPRE_Int variant, overlap, domain_type;
    HYPRE_Real schwarz_rlx_weight;
-   HYPRE_Real *values;
+   HYPRE_Real *values, val;
+
+   HYPRE_Int use_nonsymm_schwarz = 0;
+   HYPRE_Int test_ij = 0;
+   HYPRE_Int build_rbm = 0;
+   HYPRE_Int build_rbm_index = 0;
+   HYPRE_Int num_interp_vecs = 0;
+   HYPRE_Int interp_vec_variant = 0;
+   HYPRE_Int Q_max = 0;
+   HYPRE_Real Q_trunc = 0;
 
    const HYPRE_Real dt_inf = 1.e40;
    HYPRE_Real dt = dt_inf;
 
    /* parameters for BoomerAMG */
    HYPRE_Real   strong_threshold;
-   HYPRE_Real   trunc_factor = 0.0;
-   HYPRE_Real   jacobi_trunc_threshold = 0.0;
-   HYPRE_Int      P_max_elmts = 0;
+   HYPRE_Real   trunc_factor;
+   HYPRE_Real   jacobi_trunc_threshold;
+   HYPRE_Real   S_commpkg_switch = 1.0;
+   HYPRE_Real   CR_rate = 0.7;
+   HYPRE_Real   CR_strong_th = 0.0;
+   HYPRE_Int      CR_use_CG = 0;
+   HYPRE_Int      P_max_elmts = 4;
    HYPRE_Int      cycle_type;
-   HYPRE_Int      coarsen_type = 6;
-   HYPRE_Int      hybrid = 1;
+   HYPRE_Int      coarsen_type = 10;
    HYPRE_Int      measure_type = 0;
-   HYPRE_Int     *num_grid_sweeps;  
-   HYPRE_Int     *grid_relax_type;   
-   HYPRE_Int    **grid_relax_points;
+   HYPRE_Int      num_sweeps = 1;  
+   HYPRE_Int      IS_type;   
+   HYPRE_Int      num_CR_relax_steps = 2;   
+   HYPRE_Int      relax_type = -1;   
+   HYPRE_Int      add_relax_type = 18;   
+   HYPRE_Int      relax_coarse = -1;   
+   HYPRE_Int      relax_up = -1;   
+   HYPRE_Int      relax_down = -1;   
+   HYPRE_Int      relax_order = 0;   
+   HYPRE_Int      level_w = -1;
+   HYPRE_Int      level_ow = -1;
 /* HYPRE_Int	    smooth_lev; */
 /* HYPRE_Int	    smooth_rlx = 8; */
    HYPRE_Int	    smooth_type = 6;
    HYPRE_Int	    smooth_num_levels = 0;
-   HYPRE_Int      relax_default;
    HYPRE_Int      smooth_num_sweeps = 1;
-   HYPRE_Int      num_sweep = 1;
-   HYPRE_Real  *relax_weight; 
-   HYPRE_Real  *omega;
+   HYPRE_Int      coarse_threshold = 9;
+   HYPRE_Int      min_coarse_size = 0;
+/* redundant coarse grid solve */
+   HYPRE_Int      seq_threshold = 0;
+   HYPRE_Int      redundant = 0;
+/* additive versions */
+   HYPRE_Int additive = -1;
+   HYPRE_Int mult_add = -1;
+   HYPRE_Int simple = -1;
+   HYPRE_Int add_P_max_elmts = 0;
+   HYPRE_Real add_trunc_factor = 0;
+
+   HYPRE_Int    rap2=0;
+   HYPRE_Int    keepTranspose = 0;
+   HYPRE_Real   relax_wt; 
+   HYPRE_Real   add_relax_wt = 1.0; 
+   HYPRE_Real   relax_wt_level; 
+   HYPRE_Real   outer_wt;
+   HYPRE_Real   outer_wt_level;
    HYPRE_Real   tol = 1.e-8, pc_tol = 0.;
+   HYPRE_Real   atol = 0.0;
    HYPRE_Real   max_row_sum = 1.;
-   HYPRE_Int      interp_type  = 0; /* default value */
-   HYPRE_Int      post_interp_type  = 0; 
-   HYPRE_Int     *dof_func;
-   HYPRE_Int      num_functions = 1;
-   HYPRE_Int      num_paths = 1;
-   HYPRE_Int      agg_num_levels = 0;
+
+   HYPRE_Int cheby_order = 2;
+   HYPRE_Real cheby_fraction = .3;
+
    /* for CGC BM Aug 25, 2006 */
    HYPRE_Int      cgcits = 1;
-
+   /* for coordinate plotting BM Oct 24, 2006 */
+   HYPRE_Int      plot_grids = 0;
+   HYPRE_Int      coord_dim  = 3;
+   float    *coordinates = NULL;
+   char    plot_file_name[256];
+   
    /* parameters for ParaSAILS */
    HYPRE_Real   sai_threshold = 0.1;
    HYPRE_Real   sai_filter = 0.1;
@@ -168,16 +246,40 @@ main( hypre_int argc,
    HYPRE_Real   drop_tol = -1;
    HYPRE_Int      nonzeros_to_keep = -1;
 
+   /* parameters for Euclid or ILU smoother in AMG */
+   HYPRE_Real   eu_ilut = 0.0;
+   HYPRE_Real   eu_sparse_A = 0.0;
+   HYPRE_Int	    eu_bj = 0;
+   HYPRE_Int	    eu_level = -1;
+   HYPRE_Int	    eu_stats = 0;
+   HYPRE_Int	    eu_mem = 0;
+   HYPRE_Int	    eu_row_scale = 0; /* Euclid only */
+
    /* parameters for GMRES */
    HYPRE_Int	    k_dim;
-
+   /* parameters for LGMRES */
+   HYPRE_Int	    aug_dim;
    /* parameters for GSMG */
    HYPRE_Int      gsmg_samples = 5;
+   /* interpolation */
+   HYPRE_Int      interp_type  = 6; /* default value */
+   HYPRE_Int      post_interp_type  = 0; /* default value */
+   /* aggressive coarsening */
+   HYPRE_Int      agg_interp_type  = 4; /* default value */
+   HYPRE_Int      agg_P_max_elmts  = 0; /* default value */
+   HYPRE_Int      agg_P12_max_elmts  = 0; /* default value */
+   HYPRE_Real   agg_trunc_factor  = 0; /* default value */
+   HYPRE_Real   agg_P12_trunc_factor  = 0; /* default value */
 
    HYPRE_Int      print_system = 0;
 
+   HYPRE_Int rel_change = 0;
    /* begin lobpcg */
 
+   HYPRE_Int      hybrid = 1;
+   HYPRE_Int      num_sweep = 1;
+   HYPRE_Int      relax_default = 3;
+
    HYPRE_Int lobpcgFlag = 0;
    HYPRE_Int lobpcgGen = 0;
    HYPRE_Int constrained = 0;
@@ -194,7 +296,7 @@ main( hypre_int argc,
    HYPRE_Int pcgMode = 1;
    HYPRE_Real pcgTol = 1e-2;
    HYPRE_Real nonOrthF;
-   
+
    FILE* filePtr;
 
    mv_MultiVectorPtr eigenvectors = NULL;
@@ -216,11 +318,21 @@ main( hypre_int argc,
    mv_InterfaceInterpreter* interpreter;
    HYPRE_MatvecFunctions matvec_fn;
 
-   HYPRE_IJMatrix      ij_B; 
+   HYPRE_IJMatrix      ij_B;
    HYPRE_ParCSRMatrix  parcsr_B;
 
    /* end lobpcg */
 
+
+   HYPRE_Real     *nongalerk_tol = NULL;
+   HYPRE_Int       nongalerk_num_tol = 0;
+
+   HYPRE_Int *row_nums = NULL;
+   HYPRE_Int *num_cols = NULL;
+   HYPRE_Int *col_nums = NULL;
+   HYPRE_Int i_indx, j_indx, num_rows;
+   HYPRE_Real *data = NULL;
+
    /*-----------------------------------------------------------
     * Initialize some stuff
     *-----------------------------------------------------------*/
@@ -231,7 +343,7 @@ main( hypre_int argc,
    hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
    hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &myid );
 /*
-   hypre_InitMemoryDebug(myid);
+  hypre_InitMemoryDebug(myid);
 */
    /*-----------------------------------------------------------
     * Set defaults
@@ -245,7 +357,7 @@ main( hypre_int argc,
    build_src_arg_index = argc;
    build_funcs_type = 0;
    build_funcs_arg_index = argc;
-   relax_default = 3;
+   IS_type = 1;
    debug_flag = 0;
 
    solver_id = 0;
@@ -253,6 +365,8 @@ main( hypre_int argc,
    ioutdat = 3;
    poutdat = 1;
 
+   hypre_sprintf (plot_file_name,"AMGgrids.CF.dat");
+
    /*-----------------------------------------------------------
     * Parse command line
     *-----------------------------------------------------------*/
@@ -304,6 +418,23 @@ main( hypre_int argc,
          build_matrix_type      = 5;
          build_matrix_arg_index = arg_index;
       }
+      else if ( strcmp(argv[arg_index], "-vardifconv") == 0 )
+      {
+         arg_index++;
+         build_matrix_type      = 6;
+         build_matrix_arg_index = arg_index;
+      }
+      else if ( strcmp(argv[arg_index], "-rotate") == 0 )
+      {
+         arg_index++;
+         build_matrix_type      = 7;
+         build_matrix_arg_index = arg_index;
+      } 
+      else if ( strcmp(argv[arg_index], "-test_ij") == 0 )
+      {
+         arg_index++;
+         test_ij = 1;
+      }
       else if ( strcmp(argv[arg_index], "-funcsfromonefile") == 0 )
       {
          arg_index++;
@@ -326,6 +457,26 @@ main( hypre_int argc,
          arg_index++;
          sparsity_known = 2;
       }
+      else if ( strcmp(argv[arg_index], "-add") == 0 )
+      {
+         arg_index++;
+         add = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-chunk") == 0 )
+      {
+         arg_index++;
+         chunk = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-off_proc") == 0 )
+      {
+         arg_index++;
+         off_proc = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-omp") == 0 )
+      {
+         arg_index++;
+         omp_flag = atoi(argv[arg_index++]);
+      }
       else if ( strcmp(argv[arg_index], "-concrete_parcsr") == 0 )
       {
          arg_index++;
@@ -334,14 +485,20 @@ main( hypre_int argc,
       else if ( strcmp(argv[arg_index], "-solver") == 0 )
       {
          arg_index++;
-
-	 /* begin lobpcg */
-	 if ( strcmp(argv[arg_index], "none") == 0 ) {
-	   solver_id = NO_SOLVER;
-	   arg_index++;
-	 }
-	 else /* end lobpcg */
-	   solver_id = atoi(argv[arg_index++]);
+         /* begin lobpcg */
+         if ( strcmp(argv[arg_index], "none") == 0 ) {
+           solver_id = NO_SOLVER;
+           arg_index++;
+         }
+         else /* end lobpcg */
+         solver_id = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-rbm") == 0 )
+      {
+         arg_index++;
+         build_rbm      = 1;
+         num_interp_vecs = atoi(argv[arg_index++]);
+         build_rbm_index = arg_index;
       }
       else if ( strcmp(argv[arg_index], "-rhsfromfile") == 0 )
       {
@@ -355,6 +512,12 @@ main( hypre_int argc,
          build_rhs_type      = 1;
          build_rhs_arg_index = arg_index;
       }      
+      else if ( strcmp(argv[arg_index], "-rhsparcsrfile") == 0 )
+      {
+         arg_index++;
+         build_rhs_type      = 7;
+         build_rhs_arg_index = arg_index;
+      }      
       else if ( strcmp(argv[arg_index], "-rhsisone") == 0 )
       {
          arg_index++;
@@ -426,21 +589,56 @@ main( hypre_int argc,
       }    
       else if ( strcmp(argv[arg_index], "-cgc") == 0 )
       {
-        arg_index++;
-        coarsen_type      = 21;
-        cgcits            = 200;
+         arg_index++;
+         coarsen_type      = 21;
+         cgcits            = 200;
       }
       else if ( strcmp(argv[arg_index], "-cgce") == 0 )
       {
-        arg_index++;
-        coarsen_type      = 22;
-        cgcits            = 200;
+         arg_index++;
+         coarsen_type      = 22;
+         cgcits            = 200;
       }
+      else if ( strcmp(argv[arg_index], "-pmis") == 0 )
+      {
+         arg_index++;
+         coarsen_type      = 8;
+      }    
+      else if ( strcmp(argv[arg_index], "-pmis1") == 0 )
+      {
+         arg_index++;
+         coarsen_type      = 9;
+      }    
+      else if ( strcmp(argv[arg_index], "-cr1") == 0 )
+      {
+         arg_index++;
+         coarsen_type      = 98;
+      }    
+      else if ( strcmp(argv[arg_index], "-cr") == 0 )
+      {
+         arg_index++;
+         coarsen_type      = 99;
+      }    
+      else if ( strcmp(argv[arg_index], "-crcg") == 0 )
+      {
+         arg_index++;
+         CR_use_CG = atoi(argv[arg_index++]);
+      }    
+      else if ( strcmp(argv[arg_index], "-hmis") == 0 )
+      {
+         arg_index++;
+         coarsen_type      = 10;
+      }    
       else if ( strcmp(argv[arg_index], "-ruge") == 0 )
       {
          arg_index++;
          coarsen_type      = 1;
       }    
+      else if ( strcmp(argv[arg_index], "-ruge1p") == 0 )
+      {
+         arg_index++;
+         coarsen_type      = 11;
+      }    
       else if ( strcmp(argv[arg_index], "-ruge2b") == 0 )
       {
          arg_index++;
@@ -465,51 +663,51 @@ main( hypre_int argc,
       {
          arg_index++;
          coarsen_type      = 6;
-      }
-      else if ( strcmp(argv[arg_index], "-pmis") == 0 )
+      }    
+      else if ( strcmp(argv[arg_index], "-gm") == 0 )
       {
          arg_index++;
-         coarsen_type      = 8;
-      }
-      else if ( strcmp(argv[arg_index], "-pmis1") == 0 )
+         measure_type      = 1;
+      }    
+      else if ( strcmp(argv[arg_index], "-is") == 0 )
       {
          arg_index++;
-         coarsen_type      = 9;
+         IS_type = atoi(argv[arg_index++]);
       }
-      else if ( strcmp(argv[arg_index], "-hmis") == 0 )
+      else if ( strcmp(argv[arg_index], "-ncr") == 0 )
       {
          arg_index++;
-         coarsen_type      = 10;
+         num_CR_relax_steps = atoi(argv[arg_index++]);
       }
-      else if ( strcmp(argv[arg_index], "-ruge1p") == 0 )
+      else if ( strcmp(argv[arg_index], "-crth") == 0 )
       {
          arg_index++;
-         coarsen_type      = 11;
+         CR_rate = atof(argv[arg_index++]);
       }
-      else if ( strcmp(argv[arg_index], "-agg_nl") == 0 )
+      else if ( strcmp(argv[arg_index], "-crst") == 0 )
       {
          arg_index++;
-         agg_num_levels = atoi(argv[arg_index++]);
+         CR_strong_th = atof(argv[arg_index++]);
       }
-      else if ( strcmp(argv[arg_index], "-np") == 0 )
+      else if ( strcmp(argv[arg_index], "-rlx") == 0 )
       {
          arg_index++;
-         num_paths = atoi(argv[arg_index++]);
+         relax_type = atoi(argv[arg_index++]);
       }
-      else if ( strcmp(argv[arg_index], "-nohybrid") == 0 )
+      else if ( strcmp(argv[arg_index], "-rlx_coarse") == 0 )
       {
          arg_index++;
-         hybrid      = -1;
-      }    
-      else if ( strcmp(argv[arg_index], "-gm") == 0 )
+         relax_coarse = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-rlx_down") == 0 )
       {
          arg_index++;
-         measure_type      = 1;
-      }    
-      else if ( strcmp(argv[arg_index], "-rlx") == 0 )
+         relax_down = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-rlx_up") == 0 )
       {
          arg_index++;
-         relax_default = atoi(argv[arg_index++]);
+         relax_up = atoi(argv[arg_index++]);
       }
       else if ( strcmp(argv[arg_index], "-smtype") == 0 )
       {
@@ -536,16 +734,42 @@ main( hypre_int argc,
          arg_index++;
          num_functions = atoi(argv[arg_index++]);
       }
+      else if ( strcmp(argv[arg_index], "-agg_nl") == 0 )
+      {
+         arg_index++;
+         agg_num_levels = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-npaths") == 0 )
+      {
+         arg_index++;
+         num_paths = atoi(argv[arg_index++]);
+      }
       else if ( strcmp(argv[arg_index], "-ns") == 0 )
       {
          arg_index++;
-         num_sweep = atoi(argv[arg_index++]);
+         num_sweeps = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-ns_coarse") == 0 )
+      {
+         arg_index++;
+         ns_coarse = atoi(argv[arg_index++]);
       }
       else if ( strcmp(argv[arg_index], "-sns") == 0 )
       {
          arg_index++;
          smooth_num_sweeps = atoi(argv[arg_index++]);
       }
+      else if ( strcmp(argv[arg_index], "-max_iter") == 0 )
+      {
+         arg_index++;
+         max_iter = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-mg_max_iter") == 0 )
+      {
+         arg_index++;
+         mg_max_iter = atoi(argv[arg_index++]);
+      }
+
       else if ( strcmp(argv[arg_index], "-dt") == 0 )
       {
          arg_index++;
@@ -558,91 +782,92 @@ main( hypre_int argc,
          print_usage = 1;
       }
       /* begin lobpcg */
-      else if ( strcmp(argv[arg_index], "-lobpcg") == 0 ) 
-	{	      				 /* use lobpcg */
-	  arg_index++;
-	  lobpcgFlag = 1;
-	}
-      else if ( strcmp(argv[arg_index], "-gen") == 0 ) 
-	{	      				 /* generalized evp */
-	  arg_index++;
-	  lobpcgGen = 1;
-	}
-      else if ( strcmp(argv[arg_index], "-con") == 0 ) 
-	{	      				 /* constrained evp */
-	  arg_index++;
-	  constrained = 1;
-	}
+      else if ( strcmp(argv[arg_index], "-lobpcg") == 0 )
+        {                                        /* use lobpcg */
+          arg_index++;
+          lobpcgFlag = 1;
+        }
+      else if ( strcmp(argv[arg_index], "-gen") == 0 )
+        {                                        /* generalized evp */
+          arg_index++;
+          lobpcgGen = 1;
+        }
+      else if ( strcmp(argv[arg_index], "-con") == 0 )
+        {                                        /* constrained evp */
+          arg_index++;
+          constrained = 1;
+        }
       else if ( strcmp(argv[arg_index], "-orthchk") == 0 )
-	{			/* lobpcg: check orthonormality */
-	  arg_index++;
-	  checkOrtho = 1;
-	}
+        {                       /* lobpcg: check orthonormality */
+          arg_index++;
+          checkOrtho = 1;
+        }
       else if ( strcmp(argv[arg_index], "-vfromfile") == 0 )
-	{		 /* lobpcg: get initial vectors from file */
-	  arg_index++;
-	  vFromFileFlag = 1;
-	}
-      else if ( strcmp(argv[arg_index], "-vrand") == 0 ) 
-	{				/* lobpcg: block size */
-	  arg_index++;
-	  blockSize = atoi(argv[arg_index++]);
-	}
+        {                /* lobpcg: get initial vectors from file */
+          arg_index++;
+          vFromFileFlag = 1;
+        }
+      else if ( strcmp(argv[arg_index], "-vrand") == 0 )
+        {                               /* lobpcg: block size */
+          arg_index++;
+          blockSize = atoi(argv[arg_index++]);
+        }
       else if ( strcmp(argv[arg_index], "-seed") == 0 )
-	{		           /* lobpcg: seed for srand */
-	  arg_index++;
-	  lobpcgSeed = atoi(argv[arg_index++]);
-	}
-      else if ( strcmp(argv[arg_index], "-itr") == 0 ) 
-	{		     /* lobpcg: max # of iterations */
-	  arg_index++;
-	  maxIterations = atoi(argv[arg_index++]);
-	}
-      else if ( strcmp(argv[arg_index], "-verb") == 0 ) 
-	{			  /* lobpcg: verbosity level */
-	  arg_index++;
-	  verbosity = atoi(argv[arg_index++]);
-	}
+        {                          /* lobpcg: seed for srand */
+          arg_index++;
+          lobpcgSeed = atoi(argv[arg_index++]);
+        }
+      else if ( strcmp(argv[arg_index], "-itr") == 0 )
+        {                    /* lobpcg: max # of iterations */
+          arg_index++;
+          maxIterations = atoi(argv[arg_index++]);
+        }
+      else if ( strcmp(argv[arg_index], "-verb") == 0 )
+        {                         /* lobpcg: verbosity level */
+          arg_index++;
+          verbosity = atoi(argv[arg_index++]);
+        }
       else if ( strcmp(argv[arg_index], "-vout") == 0 )
-	{			      /* lobpcg: print level */
-	  arg_index++;
-	  printLevel = atoi(argv[arg_index++]);
-	}
-      else if ( strcmp(argv[arg_index], "-pcgitr") == 0 ) 
-	{		       /* lobpcg: inner pcg iterations */
-	  arg_index++;
-	  pcgIterations = atoi(argv[arg_index++]);
-	}
-      else if ( strcmp(argv[arg_index], "-pcgtol") == 0 ) 
-	{		       /* lobpcg: inner pcg iterations */
-	  arg_index++;
-	  pcgTol = atof(argv[arg_index++]);
-	}
-      else if ( strcmp(argv[arg_index], "-pcgmode") == 0 ) 
-	{		 /* lobpcg: initial guess for inner pcg */
-	  arg_index++;	      /* 0: zero, otherwise rhs */
-	  pcgMode = atoi(argv[arg_index++]);
-	}
+        {                             /* lobpcg: print level */
+          arg_index++;
+          printLevel = atoi(argv[arg_index++]);
+        }
+      else if ( strcmp(argv[arg_index], "-pcgitr") == 0 )
+        {                      /* lobpcg: inner pcg iterations */
+          arg_index++;
+          pcgIterations = atoi(argv[arg_index++]);
+        }
+      else if ( strcmp(argv[arg_index], "-pcgtol") == 0 )
+        {                      /* lobpcg: inner pcg iterations */
+          arg_index++;
+          pcgTol = atof(argv[arg_index++]);
+        }
+      else if ( strcmp(argv[arg_index], "-pcgmode") == 0 )
+        {                /* lobpcg: initial guess for inner pcg */
+          arg_index++;        /* 0: zero, otherwise rhs */
+          pcgMode = atoi(argv[arg_index++]);
+        }
       /* end lobpcg */
       else
-	{
-	  arg_index++;
-	}
+      {
+         arg_index++;
+      }
    }
+
    /* begin CGC BM Aug 25, 2006 */
    if (coarsen_type == 21 || coarsen_type == 22) {
       arg_index = 0;
       while ( (arg_index < argc) && (!print_usage) )
       {
-          if ( strcmp(argv[arg_index], "-cgcits") == 0 )
-          {
-             arg_index++;
-             cgcits = atoi(argv[arg_index++]);
-          }
-          else
-          {
-             arg_index++;
-          }
+         if ( strcmp(argv[arg_index], "-cgcits") == 0 )
+         {
+            arg_index++;
+            cgcits = atoi(argv[arg_index++]);
+         }
+         else
+         {
+            arg_index++;
+         }
       }
    }
 
@@ -655,149 +880,31 @@ main( hypre_int argc,
 
    if (solver_id == 8 || solver_id == 18)
    {
-     max_levels = 1;
+      max_levels = 1;
    }
 
    /* defaults for BoomerAMG */
    if (solver_id == 0 || solver_id == 1 || solver_id == 3 || solver_id == 5
-	|| solver_id == 9 || solver_id == 13 || solver_id == 14
- 	|| solver_id == 15 || solver_id == 20)
-   {
-   strong_threshold = 0.25;
-   trunc_factor = 0.;
-   cycle_type = 1;
-
-   num_grid_sweeps   = hypre_CTAlloc(HYPRE_Int,4);
-   grid_relax_type   = hypre_CTAlloc(HYPRE_Int,4);
-   grid_relax_points = hypre_CTAlloc(HYPRE_Int *,4);
-   relax_weight      = hypre_CTAlloc(HYPRE_Real, max_levels);
-   omega      = hypre_CTAlloc(HYPRE_Real, max_levels);
-
-   for (i=0; i < max_levels; i++)
-   {
-	relax_weight[i] = 1.;
-	omega[i] = 1.;
-   }
-
-   /* for CGNR preconditioned with Boomeramg, only relaxation scheme 0 is
-      implemented, i.e. Jacobi relaxation, and it needs to be used without
-      CF ordering */
-   if (solver_id == 5) 
-   {
-      /* fine grid */
-      relax_default = 7;
-      grid_relax_type[0] = relax_default; 
-      num_grid_sweeps[0] = num_sweep;
-      grid_relax_points[0] = hypre_CTAlloc(HYPRE_Int, num_sweep); 
-      for (i=0; i<num_sweep; i++)
-      {
-         grid_relax_points[0][i] = 0;
-      } 
-      /* down cycle */
-      grid_relax_type[1] = relax_default; 
-       num_grid_sweeps[1] = num_sweep;
-      grid_relax_points[1] = hypre_CTAlloc(HYPRE_Int, num_sweep); 
-      for (i=0; i<num_sweep; i++)
-      {
-         grid_relax_points[1][i] = 0;
-      } 
-      /* up cycle */
-      grid_relax_type[2] = relax_default; 
-      num_grid_sweeps[2] = num_sweep;
-      grid_relax_points[2] = hypre_CTAlloc(HYPRE_Int, num_sweep); 
-      for (i=0; i<num_sweep; i++)
-      {
-         grid_relax_points[2][i] = 0;
-      } 
-   }
-   else if (coarsen_type == 5)
+       || solver_id == 9 || solver_id == 13 || solver_id == 14
+       || solver_id == 15 || solver_id == 20 || solver_id == 51 || solver_id == 61)
    {
-      /* fine grid */
-      num_grid_sweeps[0] = 3;
-      grid_relax_type[0] = relax_default; 
-      grid_relax_points[0] = hypre_CTAlloc(HYPRE_Int, 3); 
-      grid_relax_points[0][0] = -2;
-      grid_relax_points[0][1] = -1;
-      grid_relax_points[0][2] = 1;
-   
-      /* down cycle */
-      num_grid_sweeps[1] = 4;
-      grid_relax_type[1] = relax_default; 
-      grid_relax_points[1] = hypre_CTAlloc(HYPRE_Int, 4); 
-      grid_relax_points[1][0] = -1;
-      grid_relax_points[1][1] = 1;
-      grid_relax_points[1][2] = -2;
-      grid_relax_points[1][3] = -2;
-   
-      /* up cycle */
-      num_grid_sweeps[2] = 4;
-      grid_relax_type[2] = relax_default; 
-      grid_relax_points[2] = hypre_CTAlloc(HYPRE_Int, 4); 
-      grid_relax_points[2][0] = -2;
-      grid_relax_points[2][1] = -2;
-      grid_relax_points[2][2] = 1;
-      grid_relax_points[2][3] = -1;
-   }
-   else
-   {   
-      /* fine grid */
-      grid_relax_type[0] = relax_default; 
-      /*num_grid_sweeps[0] = num_sweep;
-      grid_relax_points[0] = hypre_CTAlloc(HYPRE_Int, num_sweep); 
-      for (i=0; i<num_sweep; i++)
-      {
-         grid_relax_points[0][i] = 0;
-      } */
-      num_grid_sweeps[0] = 2*num_sweep;
-      grid_relax_points[0] = hypre_CTAlloc(HYPRE_Int, 2*num_sweep); 
-      for (i=0; i<2*num_sweep; i+=2)
-      {
-         grid_relax_points[0][i] = 1;
-         grid_relax_points[0][i+1] = -1;
-      } 
-
-      /* down cycle */
-      grid_relax_type[1] = relax_default; 
-      /* num_grid_sweeps[1] = num_sweep;
-      grid_relax_points[1] = hypre_CTAlloc(HYPRE_Int, num_sweep); 
-      for (i=0; i<num_sweep; i++)
-      {
-         grid_relax_points[1][i] = 0;
-      } */
-      num_grid_sweeps[1] = 2*num_sweep;
-      grid_relax_type[1] = relax_default; 
-      grid_relax_points[1] = hypre_CTAlloc(HYPRE_Int, 2*num_sweep); 
-      for (i=0; i<2*num_sweep; i+=2)
-      {
-         grid_relax_points[1][i] = 1;
-         grid_relax_points[1][i+1] = -1;
-      }
-
-      /* up cycle */
-      grid_relax_type[2] = relax_default; 
-      /* num_grid_sweeps[2] = num_sweep;
-      grid_relax_points[2] = hypre_CTAlloc(HYPRE_Int, num_sweep); 
-      for (i=0; i<num_sweep; i++)
-      {
-         grid_relax_points[2][i] = 0;
-      } */
-      num_grid_sweeps[2] = 2*num_sweep;
-      grid_relax_type[2] = relax_default; 
-      grid_relax_points[2] = hypre_CTAlloc(HYPRE_Int, 2*num_sweep); 
-      for (i=0; i<2*num_sweep; i+=2)
-      {
-         grid_relax_points[2][i] = -1;
-         grid_relax_points[2][i+1] = 1;
+      strong_threshold = 0.25;
+      trunc_factor = 0.;
+      jacobi_trunc_threshold = 0.01;
+      cycle_type = 1;
+      relax_wt = 1.;
+      outer_wt = 1.;
+
+      /* for CGNR preconditioned with Boomeramg, only relaxation scheme 0 is
+         implemented, i.e. Jacobi relaxation, and needs to be used without CF
+         ordering */
+      if (solver_id == 5) 
+      {     
+         relax_type = 0;
+         relax_order = 0;
       }
    }
 
-   /* coarsest grid */
-   num_grid_sweeps[3] = 1;
-   grid_relax_type[3] = 9;
-   grid_relax_points[3] = hypre_CTAlloc(HYPRE_Int, 1);
-   grid_relax_points[3][0] = 0;
-   }
-
    /* defaults for Schwarz */
 
    variant = 0;  /* multiplicative */
@@ -809,6 +916,9 @@ main( hypre_int argc,
 
    k_dim = 5;
 
+   /* defaults for LGMRES - should use a larger k_dim, though*/
+   aug_dim = 2;
+
    arg_index = 0;
    while (arg_index < argc)
    {
@@ -817,40 +927,68 @@ main( hypre_int argc,
          arg_index++;
          k_dim = atoi(argv[arg_index++]);
       }
+      else if ( strcmp(argv[arg_index], "-aug") == 0 )
+      {
+         arg_index++;
+         aug_dim = atoi(argv[arg_index++]);
+      }
       else if ( strcmp(argv[arg_index], "-w") == 0 )
       {
          arg_index++;
-        if (solver_id == 0 || solver_id == 1 || solver_id == 3 
-		|| solver_id == 5 || solver_id == 13 || solver_id == 14
-		|| solver_id == 15 || solver_id == 9 || solver_id == 20)
-        {
-         relax_weight[0] = atof(argv[arg_index++]);
-         for (i=1; i < max_levels; i++)
-	   relax_weight[i] = relax_weight[0];
-        }
+         relax_wt = atof(argv[arg_index++]);
       }
-      else if ( strcmp(argv[arg_index], "-om") == 0 )
+      else if ( strcmp(argv[arg_index], "-wl") == 0 )
       {
          arg_index++;
-        if (solver_id == 0 || solver_id == 1 || solver_id == 3 
-		|| solver_id == 5 || solver_id == 13 || solver_id == 14
-		|| solver_id == 15 || solver_id == 9 || solver_id == 20)
-        {
-         omega[0] = atof(argv[arg_index++]);
-         for (i=1; i < max_levels; i++)
-	   omega[i] = omega[0];
-        }
+         relax_wt_level = atof(argv[arg_index++]);
+         level_w = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-ow") == 0 )
+      {
+         arg_index++;
+         outer_wt = atof(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-owl") == 0 )
+      {
+         arg_index++;
+         outer_wt_level = atof(argv[arg_index++]);
+         level_ow = atoi(argv[arg_index++]);
       }
       else if ( strcmp(argv[arg_index], "-sw") == 0 )
       {
          arg_index++;
          schwarz_rlx_weight = atof(argv[arg_index++]);
       }
+      else if ( strcmp(argv[arg_index], "-coarse_th") == 0 )
+      {
+         arg_index++;
+         coarse_threshold  = atof(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-min_cs") == 0 )
+      {
+         arg_index++;
+         min_coarse_size  = atof(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-seq_th") == 0 )
+      {
+         arg_index++;
+         seq_threshold  = atof(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-red") == 0 )
+      {
+         arg_index++;
+         redundant  = atof(argv[arg_index++]);
+      }
       else if ( strcmp(argv[arg_index], "-th") == 0 )
       {
          arg_index++;
          strong_threshold  = atof(argv[arg_index++]);
       }
+      else if ( strcmp(argv[arg_index], "-CF") == 0 )
+      {
+         arg_index++;
+         relax_order = atoi(argv[arg_index++]);
+      }
       else if ( strcmp(argv[arg_index], "-cf") == 0 )
       {
          arg_index++;
@@ -861,6 +999,11 @@ main( hypre_int argc,
          arg_index++;
          tol  = atof(argv[arg_index++]);
       }
+      else if ( strcmp(argv[arg_index], "-atol") == 0 )
+      {
+         arg_index++;
+         atol  = atof(argv[arg_index++]);
+      }
       else if ( strcmp(argv[arg_index], "-mxrs") == 0 )
       {
          arg_index++;
@@ -886,6 +1029,41 @@ main( hypre_int argc,
          arg_index++;
          nonzeros_to_keep  = atoi(argv[arg_index++]);
       }
+      else if ( strcmp(argv[arg_index], "-ilut") == 0 )
+      {
+         arg_index++;
+         eu_ilut  = atof(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-sparseA") == 0 )
+      {
+         arg_index++;
+         eu_sparse_A  = atof(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-rowScale") == 0 )
+      {
+         arg_index++;
+         eu_row_scale  = 1;
+      }
+      else if ( strcmp(argv[arg_index], "-level") == 0 )
+      {
+         arg_index++;
+         eu_level  = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-bj") == 0 )
+      {
+         arg_index++;
+         eu_bj  = 1;
+      }
+      else if ( strcmp(argv[arg_index], "-eu_stats") == 0 )
+      {
+         arg_index++;
+         eu_stats  = 1;
+      }
+      else if ( strcmp(argv[arg_index], "-eu_mem") == 0 )
+      {
+         arg_index++;
+         eu_mem  = 1;
+      }
       else if ( strcmp(argv[arg_index], "-tr") == 0 )
       {
          arg_index++;
@@ -896,6 +1074,31 @@ main( hypre_int argc,
          arg_index++;
          P_max_elmts  = atoi(argv[arg_index++]);
       }
+      else if ( strcmp(argv[arg_index], "-interpvecvar") == 0 )
+      {
+         arg_index++;
+         interp_vec_variant  = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-Qtr") == 0 )
+      {
+         arg_index++;
+         Q_trunc  = atof(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-Qmx") == 0 )
+      {
+         arg_index++;
+         Q_max = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-jtr") == 0 )
+      {
+         arg_index++;
+         jacobi_trunc_threshold  = atof(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-Ssw") == 0 )
+      {
+         arg_index++;
+         S_commpkg_switch = atof(argv[arg_index++]);
+      }
       else if ( strcmp(argv[arg_index], "-solver_type") == 0 )
       {
          arg_index++;
@@ -916,6 +1119,11 @@ main( hypre_int argc,
          arg_index++;
          variant  = atoi(argv[arg_index++]);
       }
+      else if ( strcmp(argv[arg_index], "-use_ns") == 0 )
+      {
+         arg_index++;
+         use_nonsymm_schwarz = 1;
+      }
       else if ( strcmp(argv[arg_index], "-ov") == 0 )
       {
          arg_index++;
@@ -926,6 +1134,14 @@ main( hypre_int argc,
          arg_index++;
          domain_type  = atoi(argv[arg_index++]);
       }
+      else if ( strcmp(argv[arg_index], "-blk_sm") == 0 )
+      {
+         arg_index++;
+         smooth_num_levels = atoi(argv[arg_index++]);
+         overlap = 0;
+         smooth_type = 6;
+         domain_type = 1;
+      }
       else if ( strcmp(argv[arg_index], "-mu") == 0 )
       {
          arg_index++;
@@ -941,16 +1157,135 @@ main( hypre_int argc,
          arg_index++;
          interp_type  = atoi(argv[arg_index++]);
       }
-      else if ( strcmp(argv[arg_index], "-print") == 0 )
+      else if ( strcmp(argv[arg_index], "-agg_interp") == 0 )
       {
          arg_index++;
-         print_system = 1;
+         agg_interp_type  = atoi(argv[arg_index++]);
       }
-      else
+      else if ( strcmp(argv[arg_index], "-agg_Pmx") == 0 )
       {
          arg_index++;
+         agg_P_max_elmts  = atoi(argv[arg_index++]);
       }
-   }
+      else if ( strcmp(argv[arg_index], "-agg_P12_mx") == 0 )
+      {
+         arg_index++;
+         agg_P12_max_elmts  = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-agg_tr") == 0 )
+      {
+         arg_index++;
+         agg_trunc_factor  = atof(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-agg_P12_tr") == 0 )
+      {
+         arg_index++;
+         agg_P12_trunc_factor  = atof(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-postinterptype") == 0 )
+      {
+         arg_index++;
+         post_interp_type  = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-nodal") == 0 )
+      {
+         arg_index++;
+         nodal  = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-rel_change") == 0 )
+      {
+         arg_index++;
+         rel_change = 1;
+      }
+      else if ( strcmp(argv[arg_index], "-nodal_diag") == 0 )
+      {
+         arg_index++;
+         nodal_diag  = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-cheby_order") == 0 )
+      {
+         arg_index++;
+         cheby_order = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-cheby_fraction") == 0 )
+      {
+         arg_index++;
+         cheby_fraction = atof(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-additive") == 0 )
+      {
+         arg_index++;
+         additive  = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-mult_add") == 0 )
+      {
+         arg_index++;
+         mult_add  = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-simple") == 0 )
+      {
+         arg_index++;
+         simple  = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-add_Pmx") == 0 )
+      {
+         arg_index++;
+         add_P_max_elmts  = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-add_tr") == 0 )
+      {
+         arg_index++;
+         add_trunc_factor  = atof(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-add_rlx") == 0 )
+      {
+         arg_index++;
+         add_relax_type = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-add_w") == 0 )
+      {
+         arg_index++;
+         add_relax_wt = atof(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-rap") == 0 )
+      {
+         arg_index++;
+         rap2  = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-keepT") == 0 )
+      {
+         arg_index++;
+         keepTranspose  = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-nongalerk_tol") == 0 )
+      {
+         arg_index++;
+         nongalerk_num_tol = atoi(argv[arg_index++]);
+         nongalerk_tol = hypre_CTAlloc(HYPRE_Real, nongalerk_num_tol);
+         for (i = 0; i < nongalerk_num_tol; i++)
+            nongalerk_tol[i] = atof(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-print") == 0 )
+      {
+         arg_index++;
+         print_system = 1;
+      }
+      /* BM Oct 23, 2006 */
+      else if ( strcmp(argv[arg_index], "-plot_grids") == 0 )
+      {
+         arg_index++;
+         plot_grids = 1;
+      }
+      else if ( strcmp(argv[arg_index], "-plot_file_name") == 0 )
+      {
+         arg_index++;
+	 hypre_sprintf (plot_file_name,"%s",argv[arg_index++]);
+      }
+      else
+      {
+         arg_index++;
+      }
+   }
 
    /*-----------------------------------------------------------
     * Print usage info
@@ -969,6 +1304,7 @@ main( hypre_int argc,
       hypre_printf("matrix read from a single file (CSR format)\n");
       hypre_printf("\n");
       hypre_printf("  -laplacian [<options>] : build 5pt 2D laplacian problem (default) \n");
+      hypre_printf("  -sysL <num functions>  : build SYSTEMS laplacian 7pt operator\n");
       hypre_printf("  -9pt [<opts>]          : build 9pt 2D laplacian problem\n");
       hypre_printf("  -27pt [<opts>]         : build 27pt 3D laplacian problem\n");
       hypre_printf("  -difconv [<opts>]      : build convection-diffusion problem\n");
@@ -985,6 +1321,8 @@ main( hypre_int argc,
       hypre_printf("rhs read from multiple files (IJ format)\n");
       hypre_printf("  -rhsfromonefile        : ");
       hypre_printf("rhs read from a single file (CSR format)\n");
+      hypre_printf("  -rhsparcsrfile        :  ");
+      hypre_printf("rhs read from multiple files (ParCSR format)\n");
       hypre_printf("  -rhsrand               : rhs is random vector\n");
       hypre_printf("  -rhsisone              : rhs is vector with unit components (default)\n");
       hypre_printf("  -xisone                : solution of all ones\n");
@@ -1017,47 +1355,108 @@ main( hypre_int argc,
       hypre_printf("       20=Hybrid solver/ DiagScale, AMG \n");
       hypre_printf("       43=Euclid-PCG      44=Euclid-GMRES   \n");
       hypre_printf("       45=Euclid-BICGSTAB\n");
+      hypre_printf("       50=DS-LGMRES         51=AMG-LGMRES     \n");
+      hypre_printf("       60=DS-FlexGMRES         61=AMG-FlexGMRES     \n");
       hypre_printf("\n");
       hypre_printf("  -cljp                 : CLJP coarsening \n");
+      hypre_printf("  -cljp1                : CLJP coarsening, fixed random \n");
       hypre_printf("  -cgc                  : CGC coarsening \n");
       hypre_printf("  -cgce                 : CGC-E coarsening \n");
-      hypre_printf("  -ruge                 : Ruge coarsening (local)\n");
+      hypre_printf("  -pmis                 : PMIS coarsening \n");
+      hypre_printf("  -pmis1                : PMIS coarsening, fixed random \n");
+      hypre_printf("  -hmis                 : HMIS coarsening (default)\n");
+      hypre_printf("  -ruge                 : Ruge-Stueben coarsening (local)\n");
+      hypre_printf("  -ruge1p               : Ruge-Stueben coarsening 1st pass only(local)\n");
       hypre_printf("  -ruge3                : third pass on boundary\n");
       hypre_printf("  -ruge3c               : third pass on boundary, keep c-points\n");
-      hypre_printf("  -ruge2b               : 2nd pass is global\n");
-      hypre_printf("  -rugerlx              : relaxes special points\n");
-      hypre_printf("  -falgout              : local ruge followed by LJP\n");
-      hypre_printf("  -nohybrid             : no switch in coarsening\n");
+      hypre_printf("  -falgout              : local Ruge_Stueben followed by CLJP\n");
       hypre_printf("  -gm                   : use global measures\n");
       hypre_printf("\n");
+      hypre_printf("  -interptype  <val>    : set interpolation type\n");
+      hypre_printf("       0=Classical modified interpolation  \n");
+      hypre_printf("       1=least squares interpolation (for GSMG only)  \n");
+      hypre_printf("       0=Classical modified interpolation for hyperbolic PDEs \n");
+      hypre_printf("       3=direct interpolation with separation of weights  \n");
+      hypre_printf("       4=multipass interpolation  \n");
+      hypre_printf("       5=multipass interpolation with separation of weights  \n");
+      hypre_printf("       6=extended classical modified interpolation (default) \n");
+      hypre_printf("       7=extended (only if no common C neighbor) interpolation  \n");
+      hypre_printf("       8=standard interpolation  \n");
+      hypre_printf("       9=standard interpolation with separation of weights  \n");
+      hypre_printf("      12=FF interpolation  \n");
+      hypre_printf("      13=FF1 interpolation  \n");
+ 
+      hypre_printf("      16=use modified unknown interpolation for a system (w/unknown or hybrid approach) \n");
+      hypre_printf("      17=use non-systems interp = 6 for a system (w/unknown or hybrid approach) \n");
+      hypre_printf("      18=use non-systems interp = 8 for a system (w/unknown or hybrid approach) \n");
+      hypre_printf("      19=use non-systems interp = 0 for a system (w/unknown or hybrid approach) \n");
+      
+
+      hypre_printf("      10=classical block interpolation for nodal systems AMG\n");
+      hypre_printf("      11=classical block interpolation with diagonal blocks for nodal systems AMG\n");
+      hypre_printf("      20=same as 10, but don't add weak connect. to diag \n");
+      hypre_printf("      21=same as 11, but don't add weak connect. to diag \n");
+      hypre_printf("      22=classical block interpolation w/Ruge's variant for nodal systems AMG \n");
+      hypre_printf("      23=same as 22, but use row sums for diag scaling matrices,for nodal systems AMG \n");
+      hypre_printf("      24=direct block interpolation for nodal systems AMG\n");
+
+
+     
+      hypre_printf("\n");
       hypre_printf("  -rlx  <val>            : relaxation type\n");
       hypre_printf("       0=Weighted Jacobi  \n");
       hypre_printf("       1=Gauss-Seidel (very slow!)  \n");
-      hypre_printf("       3=Hybrid Jacobi/Gauss-Seidel  \n");
+      hypre_printf("       3=Hybrid Gauss-Seidel  \n");
+      hypre_printf("       4=Hybrid backward Gauss-Seidel  \n");
+      hypre_printf("       6=Hybrid symmetric Gauss-Seidel  \n");
+      hypre_printf("       8= symmetric L1-Gauss-Seidel  \n");
+      hypre_printf("       13= forward L1-Gauss-Seidel  \n");
+      hypre_printf("       14= backward L1-Gauss-Seidel  \n");
+      hypre_printf("       15=CG  \n");
+      hypre_printf("       16=Chebyshev  \n");
+      hypre_printf("       17=FCF-Jacobi  \n");
+      hypre_printf("       18=L1-Jacobi (may be used with -CF) \n");
+      hypre_printf("       9=Gauss elimination (use for coarsest grid only)  \n");
+      hypre_printf("       99=Gauss elimination with pivoting (use for coarsest grid only)  \n");
+      hypre_printf("       20= Nodal Weighted Jacobi (for systems only) \n");
+      hypre_printf("       23= Nodal Hybrid Jacobi/Gauss-Seidel (for systems only) \n");
+      hypre_printf("       26= Nodal Hybrid Symmetric Gauss-Seidel  (for systems only)\n");
+      hypre_printf("       29= Nodal Gauss elimination (use for coarsest grid only)  \n");
+      hypre_printf("  -rlx_coarse  <val>       : set relaxation type for coarsest grid\n");
+      hypre_printf("  -rlx_down    <val>       : set relaxation type for down cycle\n");
+      hypre_printf("  -rlx_up      <val>       : set relaxation type for up cycle\n");
+      hypre_printf("  -cheby_order  <val> : set order (1-4) for Chebyshev poly. smoother (default is 2)\n");
+      hypre_printf("  -cheby_fraction <val> : fraction of the spectrum for Chebyshev poly. smoother (default is .3)\n");
+      hypre_printf("  -nodal  <val>            : nodal system type\n");
+      hypre_printf("       0 = Unknown approach \n");
+      hypre_printf("       1 = Frobenius norm  \n");
+      hypre_printf("       2 = Sum of Abs.value of elements  \n");
+      hypre_printf("       3 = Largest magnitude element (includes its sign)  \n");
+      hypre_printf("       4 = Inf. norm  \n");
+      hypre_printf("       5 = One norm  (note: use with block version only) \n");
+      hypre_printf("       6 = Sum of all elements in block  \n");
+      hypre_printf("  -nodal_diag <val>        :how to treat diag elements\n");
+      hypre_printf("       0 = no special treatment \n");
+      hypre_printf("       1 = make diag = neg.sum of the off_diag  \n");
+      hypre_printf("       2 = make diag = neg. of diag \n");
       hypre_printf("  -ns <val>              : Use <val> sweeps on each level\n");
       hypre_printf("                           (default C/F down, F/C up, F/C fine\n");
+      hypre_printf("  -ns_coarse  <val>       : set no. of sweeps for coarsest grid\n");
+      hypre_printf("  -ns_down    <val>       : set no. of sweeps for down cycle\n");
+      hypre_printf("  -ns_up      <val>       : set no. of sweeps for up cycle\n");
       hypre_printf("\n"); 
       hypre_printf("  -mu   <val>            : set AMG cycles (1=V, 2=W, etc.)\n"); 
       hypre_printf("  -th   <val>            : set AMG threshold Theta = val \n");
       hypre_printf("  -tr   <val>            : set AMG interpolation truncation factor = val \n");
-      hypre_printf("  -Pmx  <val>            : set maximal no. of elmts per row for AMG interpolation \n");
+      hypre_printf("  -Pmx  <val>            : set maximal no. of elmts per row for AMG interpolation (default: 4)\n");
       hypre_printf("  -jtr  <val>            : set truncation threshold for Jacobi interpolation = val \n");
-
+      hypre_printf("  -Ssw  <val>            : set S-commpkg-switch = val \n");
       hypre_printf("  -mxrs <val>            : set AMG maximum row sum threshold for dependency weakening \n");
       hypre_printf("  -nf <val>              : set number of functions for systems AMG\n");
       hypre_printf("  -numsamp <val>         : set number of sample vectors for GSMG\n");
-      hypre_printf("  -interptype <val>      : set to 1 to get LS interpolation (for GSMG only)\n");
-      hypre_printf("                         : set to 2 to get interpolation for hyperbolic equations\n");
-      hypre_printf("                         : set to 3 to get direct interpolation (with weight separation)\n");
-      hypre_printf("                         : set to 4 to get multipass interpolation\n");
-      hypre_printf("                         : set to 5 to get multipass interpolation with weight separation\n");
-      hypre_printf("                         : set to 6 to get extended interpolation\n");
-      hypre_printf("                         : set to 7 to get FF interpolation\n");
-      hypre_printf("                         : set to 8 to get standard interpolation\n");
-      hypre_printf("                         : set to 9 to get standard interpolation with weight separation\n");
-      hypre_printf("                         : set to 10 for nodal standard interpolation (for systems only) \n");
-      hypre_printf("                         : set to 11 for diagonal nodal standard interpolation (for systems only) \n");
+    
       hypre_printf("  -postinterptype <val>  : invokes <val> no. of Jacobi interpolation steps after main interpolation\n");
+      hypre_printf("\n");
       hypre_printf("  -cgcitr <val>          : set maximal number of coarsening iterations for CGC\n");
       hypre_printf("  -solver_type <val>     : sets solver within Hybrid solver\n");
       hypre_printf("                         : 1  PCG  (default)\n");
@@ -1066,11 +1465,25 @@ main( hypre_int argc,
      
       hypre_printf("  -w   <val>             : set Jacobi relax weight = val\n");
       hypre_printf("  -k   <val>             : dimension Krylov space for GMRES\n");
+      hypre_printf("  -aug   <val>           : number of augmentation vectors for LGMRES (-k indicates total approx space size)\n");
+
       hypre_printf("  -mxl  <val>            : maximum number of levels (AMG, ParaSAILS)\n");
       hypre_printf("  -tol  <val>            : set solver convergence tolerance = val\n");
+      hypre_printf("  -atol  <val>           : set solver absolute convergence tolerance = val\n");
+      hypre_printf("  -max_iter  <val>       : set max iterations\n");
+      hypre_printf("  -mg_max_iter  <val>    : set max iterations for mg solvers\n");
+      hypre_printf("  -agg_nl  <val>         : set number of aggressive coarsening levels (default:0)\n");
+      hypre_printf("  -np  <val>             : set number of paths of length 2 for aggr. coarsening\n");
       hypre_printf("\n");
       hypre_printf("  -sai_th   <val>        : set ParaSAILS threshold = val \n");
       hypre_printf("  -sai_filt <val>        : set ParaSAILS filter = val \n");
+      hypre_printf("\n");
+      hypre_printf("  -level   <val>         : set k in ILU(k) for Euclid \n");
+      hypre_printf("  -bj <val>              : enable block Jacobi ILU for Euclid \n");
+      hypre_printf("  -ilut <val>            : set drop tolerance for ILUT in Euclid\n");
+      hypre_printf("                           Note ILUT is sequential only!\n");
+      hypre_printf("  -sparseA <val>         : set drop tolerance in ILU(k) for Euclid \n");
+      hypre_printf("  -rowScale <val>        : enable row scaling in Euclid \n");
       hypre_printf("\n");  
       hypre_printf("  -drop_tol  <val>       : set threshold for dropping in PILUT\n");
       hypre_printf("  -nonzeros_to_keep <val>: number of nonzeros in each row to keep\n");
@@ -1084,7 +1497,6 @@ main( hypre_int argc,
       hypre_printf("\n");
       hypre_printf("  -print                 : print out the system\n");
       hypre_printf("\n");
-
       /* begin lobpcg */
 
       hypre_printf("LOBPCG options:\n");
@@ -1108,7 +1520,7 @@ main( hypre_int argc,
       hypre_printf("                            number generator(default seed is based\n");
       hypre_printf("                            on the time of the run)\n");
       hypre_printf("\n");
-      hypre_printf("  -vfromfile              : read initial vectors from files\n"); 
+      hypre_printf("  -vfromfile              : read initial vectors from files\n");
       hypre_printf("                            vectors.i.j where i is vector number\n");
       hypre_printf("                            and j is processor number\n");
       hypre_printf("\n");
@@ -1148,6 +1560,19 @@ main( hypre_int argc,
 
       /* end lobpcg */
 
+      hypre_printf("  -plot_grids            : print out information for plotting the grids\n");
+      hypre_printf("  -plot_file_name <val>  : file name for plotting output\n");
+      hypre_printf("\n");
+      hypre_printf("  -smtype <val>      :smooth type\n");
+      hypre_printf("  -smlv <val>        :smooth num levels\n");
+      hypre_printf("  -ov <val>          :over lap:\n");
+      hypre_printf("  -dom <val>         :domain type\n");
+      hypre_printf("  -use_ns            : use non-symm schwarz smoother\n");
+      hypre_printf("  -var <val>         : schwarz smoother variant (0-3) \n");
+      hypre_printf("  -blk_sm <val>      : same as '-smtype 6 -ov 0 -dom 1 -smlv <val>'\n");
+      hypre_printf("  -nongalerk_tol <val> <list>    : specify the NonGalerkin drop tolerance\n");
+      hypre_printf("                                   and list contains the values, where last value\n");
+      hypre_printf("                                   in list is repeated if val < num_levels in AMG\n");
       exit(1);
    }
 
@@ -1171,10 +1596,17 @@ main( hypre_int argc,
       hypre_printf("  Dirichlet 0 BCs are implicit in the spatial operator\n");
    }
 
+   time_index = hypre_InitializeTiming("Spatial Operator");
+   hypre_BeginTiming(time_index);
    if ( build_matrix_type == -1 )
    {
-      HYPRE_IJMatrixRead( argv[build_matrix_arg_index], comm,
-                          HYPRE_PARCSR, &ij_A );
+      ierr = HYPRE_IJMatrixRead( argv[build_matrix_arg_index], comm,
+                                 HYPRE_PARCSR, &ij_A );
+      if (ierr)
+      {
+         hypre_printf("ERROR: Problem reading in the system matrix!\n");
+         exit(1);
+      }
    }
    else if ( build_matrix_type == 0 )
    {
@@ -1182,8 +1614,8 @@ main( hypre_int argc,
    }
    else if ( build_matrix_type == 1 )
    {
-      BuildParFromOneFile2(argc, argv, build_matrix_arg_index, num_functions,
-	&parcsr_A);
+      BuildParFromOneFile(argc, argv, build_matrix_arg_index, num_functions,
+                          &parcsr_A);
    }
    else if ( build_matrix_type == 2 )
    {
@@ -1201,149 +1633,215 @@ main( hypre_int argc,
    {
       BuildParDifConv(argc, argv, build_matrix_arg_index, &parcsr_A);
    }
+   else if ( build_matrix_type == 6 )
+   {
+      BuildParVarDifConv(argc, argv, build_matrix_arg_index, &parcsr_A, &b);
+      /*HYPRE_ParCSRMatrixPrint(parcsr_A,"mat100");*/
+   }
+   else if ( build_matrix_type == 7 )
+   {
+      BuildParRotate7pt(argc, argv, build_matrix_arg_index, &parcsr_A);
+   }
+
    else
    {
       hypre_printf("You have asked for an unsupported problem with\n");
       hypre_printf("build_matrix_type = %d.\n", build_matrix_type);
       return(-1);
    }
-
-   time_index = hypre_InitializeTiming("Spatial operator");
-   hypre_BeginTiming(time_index);
+   /* BM Oct 23, 2006 */
+   if (plot_grids)
+   {
+      if (build_matrix_type > 1 &&  build_matrix_type < 8)
+         BuildParCoordinates (argc, argv, build_matrix_arg_index, 
+                              &coord_dim, &coordinates);
+      else
+      {
+         hypre_printf("Warning: coordinates are not yet printed for build_matrix_type = %d.\n", build_matrix_type);
+      }
+   }
 
    if (build_matrix_type < 0)
    {
-     ierr += HYPRE_IJMatrixGetObject( ij_A, &object);
-     parcsr_A = (HYPRE_ParCSRMatrix) object;
-
-     ierr = HYPRE_ParCSRMatrixGetLocalRange( parcsr_A,
-               &first_local_row, &last_local_row ,
-               &first_local_col, &last_local_col );
-
-     local_num_rows = last_local_row - first_local_row + 1;
-     local_num_cols = last_local_col - first_local_col + 1;
+      ierr = HYPRE_IJMatrixGetLocalRange( ij_A,
+                                          &first_local_row, &last_local_row ,
+                                          &first_local_col, &last_local_col );
+
+      local_num_rows = last_local_row - first_local_row + 1;
+      local_num_cols = last_local_col - first_local_col + 1;
+      ierr += HYPRE_IJMatrixGetObject( ij_A, &object);
+      parcsr_A = (HYPRE_ParCSRMatrix) object;
    }
    else
    {
+      /*-----------------------------------------------------------
+       * Copy the parcsr matrix into the IJMatrix through interface calls
+       *-----------------------------------------------------------*/
+      ierr = HYPRE_ParCSRMatrixGetLocalRange( parcsr_A,
+                                              &first_local_row, &last_local_row ,
+                                              &first_local_col, &last_local_col );
 
-     /*-----------------------------------------------------------
-      * Copy the parcsr matrix into the IJMatrix through interface calls
-      *-----------------------------------------------------------*/
- 
-     ierr = HYPRE_ParCSRMatrixGetLocalRange( parcsr_A,
-               &first_local_row, &last_local_row ,
-               &first_local_col, &last_local_col );
+      local_num_rows = last_local_row - first_local_row + 1;
+      local_num_cols = last_local_col - first_local_col + 1;
+   }
+   hypre_EndTiming(time_index);
+   hypre_PrintTiming("Generate Matrix", hypre_MPI_COMM_WORLD);
+   hypre_FinalizeTiming(time_index);
+   hypre_ClearTiming();
 
-     local_num_rows = last_local_row - first_local_row + 1;
-     local_num_cols = last_local_col - first_local_col + 1;
-     ierr += HYPRE_ParCSRMatrixGetDims( parcsr_A, &M, &N );
+   /* Check the ij interface - not necessary if one just wants to test solvers */
+   if (test_ij && build_matrix_type > -1)
+   {
+      HYPRE_Int mx_size = 5; 
+      time_index = hypre_InitializeTiming("Generate IJ matrix");
+      hypre_BeginTiming(time_index);
 
-     ierr += HYPRE_IJMatrixCreate( comm, first_local_row, last_local_row,
-                                   first_local_col, last_local_col,
-                                   &ij_A );
+      ierr += HYPRE_ParCSRMatrixGetDims( parcsr_A, &M, &N );
 
-     ierr += HYPRE_IJMatrixSetObjectType( ij_A, HYPRE_PARCSR );
-  
+      ierr += HYPRE_IJMatrixCreate( comm, first_local_row, last_local_row,
+                                    first_local_col, last_local_col,
+                                    &ij_A );
 
-/* the following shows how to build an IJMatrix if one has only an
-   estimate for the row sizes */
+      ierr += HYPRE_IJMatrixSetObjectType( ij_A, HYPRE_PARCSR );
+      num_rows = local_num_rows;
+      if (off_proc)
+      {
+	 if (myid != num_procs-1) num_rows++;
+	 if (myid) num_rows++;
+      }
+      /* The following shows how to build an IJMatrix if one has only an
+         estimate for the row sizes */
+     row_nums = hypre_CTAlloc(HYPRE_Int, num_rows);
+     num_cols = hypre_CTAlloc(HYPRE_Int, num_rows);
      if (sparsity_known == 1)
      {   
-/*  build IJMatrix using exact row_sizes for diag and offdiag */
-
-       diag_sizes = hypre_CTAlloc(HYPRE_Int, local_num_rows);
-       offdiag_sizes = hypre_CTAlloc(HYPRE_Int, local_num_rows);
-       local_row = 0;
-       for (i=first_local_row; i<= last_local_row; i++)
-       {
-         ierr += HYPRE_ParCSRMatrixGetRow( parcsr_A, i, &size, 
-                                           &col_inds, &values );
- 
-         for (j=0; j < size; j++)
+         diag_sizes = hypre_CTAlloc(HYPRE_Int, local_num_rows);
+         offdiag_sizes = hypre_CTAlloc(HYPRE_Int, local_num_rows);
+     }   
+     else
+     {   
+         size = 5;
+         if (sparsity_known == 0)
+         {
+            if (build_matrix_type == 2) size = 7;
+            if (build_matrix_type == 3) size = 9;
+            if (build_matrix_type == 4) size = 27;
+         }
+	 row_sizes = hypre_CTAlloc(HYPRE_Int, num_rows);
+	 for (i=0; i < num_rows; i++)
+	    row_sizes[i] = size;
+     }   
+     local_row = 0;
+     if (build_matrix_type == 2) mx_size = 7;
+     if (build_matrix_type == 3) mx_size = 9;
+     if (build_matrix_type == 4) mx_size = 27;
+     col_nums = hypre_CTAlloc(HYPRE_Int, mx_size*num_rows);
+     data = hypre_CTAlloc(HYPRE_Real, mx_size*num_rows);
+     i_indx = 0;
+     j_indx = 0;
+     if (off_proc && myid)
+     {
+	 num_cols[i_indx] = 2;
+	 row_nums[i_indx++] = first_local_row-1;
+         col_nums[j_indx] = first_local_row-1;
+         data[j_indx++] = 6.;
+         col_nums[j_indx] = first_local_row-2;
+         data[j_indx++] = -1;
+     }
+     for (i=0; i < local_num_rows; i++)
+     {
+         row_nums[i_indx] = first_local_row +i;
+         ierr += HYPRE_ParCSRMatrixGetRow( parcsr_A, first_local_row+i, &size, 
+                                             &col_inds, &values);
+	 num_cols[i_indx++] = size;
+         for (j = 0; j < size; j++)
          {
-           if (col_inds[j] < first_local_row || col_inds[j] > last_local_row)
-             offdiag_sizes[local_row]++;
-           else
-             diag_sizes[local_row]++;
+            col_nums[j_indx] = col_inds[j];
+            data[j_indx++] = values[j];
+            if (sparsity_known == 1)
+            {
+               if (col_inds[j] < first_local_row || col_inds[j] > last_local_row)
+                  offdiag_sizes[local_row]++;
+               else
+                  diag_sizes[local_row]++;
+            }
          }
          local_row++;
-         ierr += HYPRE_ParCSRMatrixRestoreRow( parcsr_A, i, &size, 
-                                               &col_inds, &values );
-       }
-       ierr += HYPRE_IJMatrixSetDiagOffdSizes( ij_A, 
-                                        (const HYPRE_Int *) diag_sizes,
-                                        (const HYPRE_Int *) offdiag_sizes );
-       hypre_TFree(diag_sizes);
-       hypre_TFree(offdiag_sizes);
-     
-       ierr = HYPRE_IJMatrixInitialize( ij_A );
-
-       for (i=first_local_row; i<= last_local_row; i++)
-       {
-         ierr += HYPRE_ParCSRMatrixGetRow( parcsr_A, i, &size,
-                                           &col_inds, &values );
-
-         ierr += HYPRE_IJMatrixSetValues( ij_A, 1, &size, &i,
-                                          (const HYPRE_Int *) col_inds,
-                                          (const HYPRE_Real *) values );
-
-         ierr += HYPRE_ParCSRMatrixRestoreRow( parcsr_A, i, &size,
-                                               &col_inds, &values );
-       }
+         ierr += HYPRE_ParCSRMatrixRestoreRow( parcsr_A, first_local_row+i, &size, 
+                                                 &col_inds, &values );
      }
-     else
+     if (off_proc && myid != num_procs-1)
      {
-       row_sizes = hypre_CTAlloc(HYPRE_Int, local_num_rows);
-
-       size = 5; /* this is in general too low, and supposed to test
-                    the capability of the reallocation of the interface */ 
-
-       if (sparsity_known == 0) /* tries a more accurate estimate of the
-                                    storage */
-       {
-         if (build_matrix_type == 2) size = 7;
-         if (build_matrix_type == 3) size = 9;
-         if (build_matrix_type == 4) size = 27;
-       }
-
-       for (i=0; i < local_num_rows; i++)
-         row_sizes[i] = size;
-
-       ierr = HYPRE_IJMatrixSetRowSizes ( ij_A, (const HYPRE_Int *) row_sizes );
-
-       hypre_TFree(row_sizes);
+	 num_cols[i_indx] = 2;
+	 row_nums[i_indx++] = last_local_row+1;
+         col_nums[j_indx] = last_local_row+2;
+         data[j_indx++] = -1.;
+         col_nums[j_indx] = last_local_row+1;
+         data[j_indx++] = 6;
+     }
 
-       ierr = HYPRE_IJMatrixInitialize( ij_A );
+         /*ierr += HYPRE_IJMatrixSetRowSizes ( ij_A, (const HYPRE_Int *) num_cols );*/
+     if (sparsity_known == 1)
+        ierr += HYPRE_IJMatrixSetDiagOffdSizes( ij_A, (const HYPRE_Int *) diag_sizes,
+                                            (const HYPRE_Int *) offdiag_sizes );
+     else
+        ierr = HYPRE_IJMatrixSetRowSizes ( ij_A, (const HYPRE_Int *) row_sizes );
 
-       /* Loop through all locally stored rows and insert them into ij_matrix */
-       for (i=first_local_row; i<= last_local_row; i++)
-       {
-         ierr += HYPRE_ParCSRMatrixGetRow( parcsr_A, i, &size,
-                                           &col_inds, &values );
+     ierr += HYPRE_IJMatrixInitialize( ij_A );
 
-         ierr += HYPRE_IJMatrixSetValues( ij_A, 1, &size, &i,
-                                          (const HYPRE_Int *) col_inds,
-                                          (const HYPRE_Real *) values );
+     if (omp_flag) HYPRE_IJMatrixSetOMPFlag(ij_A, 1);
 
-         ierr += HYPRE_ParCSRMatrixRestoreRow( parcsr_A, i, &size,
-                                               &col_inds, &values );
-       }
+     if (chunk)
+     {
+         if (add)
+	    ierr += HYPRE_IJMatrixAddToValues(ij_A, num_rows, num_cols, row_nums, 
+					(const HYPRE_Int *) col_nums, 
+					(const HYPRE_Real *) data);
+         else
+            ierr += HYPRE_IJMatrixSetValues(ij_A, num_rows, num_cols, row_nums, 
+					(const HYPRE_Int *) col_nums, 
+					(const HYPRE_Real *) data);
      }
+     else 
+     {
+         j_indx = 0;
+         for (i=0; i < num_rows; i++)
+         {
+            if (add)
+               ierr += HYPRE_IJMatrixAddToValues( ij_A, 1, &num_cols[i], &row_nums[i],
+                                             (const HYPRE_Int *) &col_nums[j_indx],
+                                             (const HYPRE_Real *) &data[j_indx] );
+            else
+               ierr += HYPRE_IJMatrixSetValues( ij_A, 1, &num_cols[i], &row_nums[i],
+                                             (const HYPRE_Int *) &col_nums[j_indx],
+                                             (const HYPRE_Real *) &data[j_indx] );
+            j_indx += num_cols[i];
+         }
+     }
+     hypre_TFree(col_nums);
+     hypre_TFree(data);
+     hypre_TFree(row_nums);
+     hypre_TFree(num_cols); 
+     if (sparsity_known == 1)
+     {
+         hypre_TFree(diag_sizes);
+         hypre_TFree(offdiag_sizes);
+     }
+     else
+         hypre_TFree(row_sizes);
 
-     ierr += HYPRE_IJMatrixAssemble( ij_A );
-
-   }
+      ierr += HYPRE_IJMatrixAssemble( ij_A );
 
-   hypre_EndTiming(time_index);
-   hypre_PrintTiming("IJ Matrix Setup", hypre_MPI_COMM_WORLD);
-   hypre_FinalizeTiming(time_index);
-   hypre_ClearTiming();
+      hypre_EndTiming(time_index);
+      hypre_PrintTiming("IJ Matrix Setup", hypre_MPI_COMM_WORLD);
+      hypre_FinalizeTiming(time_index);
+      hypre_ClearTiming();
    
-   if (ierr)
-   {
-     hypre_printf("Error in driver building IJMatrix from parcsr matrix. \n");
-     return(-1);
-   }
+      if (ierr)
+      {
+         hypre_printf("Error in driver building IJMatrix from parcsr matrix. \n");
+         return(-1); 
+      }
 
    /* This is to emphasize that one can IJMatrixAddToValues after an
       IJMatrixRead or an IJMatrixAssemble.  After an IJMatrixRead,
@@ -1351,41 +1849,77 @@ main( hypre_int argc,
       not changed somehow.  If one has not used IJMatrixRead, one has
       the opportunity to IJMatrixAddTo before a IJMatrixAssemble. */
 
+      ncols    = hypre_CTAlloc(HYPRE_Int, last_local_row - first_local_row + 1);
+      rows     = hypre_CTAlloc(HYPRE_Int, last_local_row - first_local_row + 1);
+      col_inds = hypre_CTAlloc(HYPRE_Int, last_local_row - first_local_row + 1);
+      values   = hypre_CTAlloc(HYPRE_Real, last_local_row - first_local_row + 1);
 
-   if (build_matrix_type > -1)
-   {
-       ierr = HYPRE_IJMatrixInitialize( ij_A );
-
-       /* Loop through all locally stored rows and insert them into ij_matrix */
-       for (i=first_local_row; i<= last_local_row; i++)
-       {
-         ierr += HYPRE_ParCSRMatrixGetRow( parcsr_A, i, &size,
-                                           &col_inds, &values );
+      if (dt < dt_inf)
+         val = 1./dt;
+      else 
+         val = 0.;   /* Use zero to avoid unintentional loss of significance */
 
-         ierr += HYPRE_IJMatrixSetValues( ij_A, 1, &size, &i,
-                                          (const HYPRE_Int *) col_inds,
-                                          (const HYPRE_Real *) values );
+      for (i = first_local_row; i <= last_local_row; i++)
+      {
+         j = i - first_local_row;
+         rows[j] = i;
+         ncols[j] = 1;
+         col_inds[j] = i;
+         values[j] = val;
+      }
+      
+      ierr += HYPRE_IJMatrixAddToValues( ij_A,
+                                      local_num_rows,
+                                      ncols, rows,
+                                      (const HYPRE_Int *) col_inds,
+                                      (const HYPRE_Real *) values );
 
-         ierr += HYPRE_ParCSRMatrixRestoreRow( parcsr_A, i, &size,
-                                               &col_inds, &values );
-       }
+      hypre_TFree(values);
+      hypre_TFree(col_inds);
+      hypre_TFree(rows);
+      hypre_TFree(ncols);
 
    /* If sparsity pattern is not changed since last IJMatrixAssemble call,
       this should be a no-op */
 
-   ierr += HYPRE_IJMatrixAssemble( ij_A );
+      ierr += HYPRE_IJMatrixAssemble( ij_A );
 
    /*-----------------------------------------------------------
     * Fetch the resulting underlying matrix out
     *-----------------------------------------------------------*/
+     if (build_matrix_type > -1)
+         ierr += HYPRE_ParCSRMatrixDestroy(parcsr_A);
 
-     ierr += HYPRE_ParCSRMatrixDestroy(parcsr_A);
+     ierr += HYPRE_IJMatrixGetObject( ij_A, &object);
+     parcsr_A = (HYPRE_ParCSRMatrix) object;
 
-   ierr += HYPRE_IJMatrixGetObject( ij_A, &object);
-   parcsr_A = (HYPRE_ParCSRMatrix) object;
    }
 
    /*-----------------------------------------------------------
+    * Set up the interp vector
+    *-----------------------------------------------------------*/
+
+   if ( build_rbm)
+   {
+      char new_file_name[80];
+      /* RHS */
+      interp_vecs = hypre_CTAlloc(HYPRE_ParVector,num_interp_vecs);
+      ij_rbm = hypre_CTAlloc(HYPRE_IJVector,num_interp_vecs);
+      for (i=0; i < num_interp_vecs; i++)
+      {
+         hypre_sprintf(new_file_name, "%s.%d", argv[build_rbm_index],i);
+         ierr = HYPRE_IJVectorRead( new_file_name, hypre_MPI_COMM_WORLD, 
+                                 HYPRE_PARCSR, &ij_rbm[i] );
+         ierr = HYPRE_IJVectorGetObject( ij_rbm[i], &object );
+         interp_vecs[i] = (HYPRE_ParVector) object;
+      }
+      if (ierr)
+      {
+         hypre_printf("ERROR: Problem reading in rbm!\n");
+         exit(1);
+      }
+   }
+   /*-----------------------------------------------------------
     * Set up the RHS and initial guess
     *-----------------------------------------------------------*/
 
@@ -1396,17 +1930,22 @@ main( hypre_int argc,
    {
       if (myid == 0)
       {
-        hypre_printf("  RHS vector read from file %s\n", argv[build_rhs_arg_index]);
-        hypre_printf("  Initial guess is 0\n");
+         hypre_printf("  RHS vector read from file %s\n", argv[build_rhs_arg_index]);
+         hypre_printf("  Initial guess is 0\n");
       }
 
-/* RHS */
+      /* RHS */
       ierr = HYPRE_IJVectorRead( argv[build_rhs_arg_index], hypre_MPI_COMM_WORLD, 
                                  HYPRE_PARCSR, &ij_b );
+      if (ierr)
+      {
+         hypre_printf("ERROR: Problem reading in the right-hand-side!\n");
+         exit(1);
+      }
       ierr = HYPRE_IJVectorGetObject( ij_b, &object );
       b = (HYPRE_ParVector) object;
 
-/* Initial guess */
+      /* Initial guess */
       HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, first_local_col, last_local_col, &ij_x);
       HYPRE_IJVectorSetObjectType(ij_x, HYPRE_PARCSR);
       HYPRE_IJVectorInitialize(ij_x);
@@ -1422,23 +1961,68 @@ main( hypre_int argc,
    }
    else if ( build_rhs_type == 1 )
    {
+
+#if 0
       hypre_printf("build_rhs_type == 1 not currently implemented\n");
       return(-1);
+#else
+      /* RHS - this has not been tested for multiple processors*/
+      BuildRhsParFromOneFile(argc, argv, build_rhs_arg_index, NULL, &b);
+
+      hypre_printf("  Initial guess is 0\n");
+
+      ij_b = NULL;
+
+      /* initial guess */ 
+      HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, first_local_col, last_local_col, &ij_x);
+      HYPRE_IJVectorSetObjectType(ij_x, HYPRE_PARCSR);
+      HYPRE_IJVectorInitialize(ij_x);
+      
+      values = hypre_CTAlloc(HYPRE_Real, local_num_cols);
+      for (i = 0; i < local_num_cols; i++)
+         values[i] = 0.;
+      HYPRE_IJVectorSetValues(ij_x, local_num_cols, NULL, values);
+      hypre_TFree(values);
+      
+      ierr = HYPRE_IJVectorGetObject( ij_x, &object );
+      x = (HYPRE_ParVector) object;
 
-#if 0
-/* RHS */
-      BuildRhsParFromOneFile2(argc, argv, build_rhs_arg_index, part_b, &b);
 #endif
    }
+   else if ( build_rhs_type == 7 )
+   {
+
+      /* rhs */ 
+      BuildParRhsFromFile(argc, argv, build_rhs_arg_index, &b);
+
+      hypre_printf("  Initial guess is 0\n");
+
+      ij_b = NULL;
+
+      /* initial guess */ 
+      HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, first_local_col, last_local_col, &ij_x);
+      HYPRE_IJVectorSetObjectType(ij_x, HYPRE_PARCSR);
+      HYPRE_IJVectorInitialize(ij_x);
+      
+      values = hypre_CTAlloc(HYPRE_Real, local_num_cols);
+      for (i = 0; i < local_num_cols; i++)
+         values[i] = 0.;
+      HYPRE_IJVectorSetValues(ij_x, local_num_cols, NULL, values);
+      hypre_TFree(values);
+      
+      ierr = HYPRE_IJVectorGetObject( ij_x, &object );
+      x = (HYPRE_ParVector) object;
+   }
+
    else if ( build_rhs_type == 2 )
    {
       if (myid == 0)
       {
-        hypre_printf("  RHS vector has unit components\n");
-        hypre_printf("  Initial guess is 0\n");
+         hypre_printf("  RHS vector has unit components\n");
+         hypre_printf("  Initial guess is 0\n");
       }
 
-/* RHS */
+      /* RHS */
       HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, first_local_row, last_local_row, &ij_b);
       HYPRE_IJVectorSetObjectType(ij_b, HYPRE_PARCSR);
       HYPRE_IJVectorInitialize(ij_b);
@@ -1452,7 +2036,7 @@ main( hypre_int argc,
       ierr = HYPRE_IJVectorGetObject( ij_b, &object );
       b = (HYPRE_ParVector) object;
 
-/* Initial guess */
+      /* Initial guess */
       HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, first_local_col, last_local_col, &ij_x);
       HYPRE_IJVectorSetObjectType(ij_x, HYPRE_PARCSR);
       HYPRE_IJVectorInitialize(ij_x);
@@ -1470,28 +2054,28 @@ main( hypre_int argc,
    {
       if (myid == 0)
       {
-        hypre_printf("  RHS vector has random components and unit 2-norm\n");
-        hypre_printf("  Initial guess is 0\n");
+         hypre_printf("  RHS vector has random components and unit 2-norm\n");
+         hypre_printf("  Initial guess is 0\n");
       }
 
-/* RHS */
+      /* RHS */
       HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, first_local_row, last_local_row, &ij_b); 
       HYPRE_IJVectorSetObjectType(ij_b, HYPRE_PARCSR);
       HYPRE_IJVectorInitialize(ij_b);
       ierr = HYPRE_IJVectorGetObject( ij_b, &object );
       b = (HYPRE_ParVector) object;
 
-/* For purposes of this test, HYPRE_ParVector functions are used, but these are 
-   not necessary.  For a clean use of the interface, the user "should"
-   modify components of ij_x by using functions HYPRE_IJVectorSetValues or
-   HYPRE_IJVectorAddToValues */
+      /* For purposes of this test, HYPRE_ParVector functions are used, but
+         these are not necessary.  For a clean use of the interface, the user
+         "should" modify components of ij_x by using functions
+         HYPRE_IJVectorSetValues or HYPRE_IJVectorAddToValues */
 
       HYPRE_ParVectorSetRandomValues(b, 22775);
       HYPRE_ParVectorInnerProd(b,b,&norm);
       norm = 1./sqrt(norm);
       ierr = HYPRE_ParVectorScale(norm, b);      
 
-/* Initial guess */
+      /* Initial guess */
       HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, first_local_col, last_local_col, &ij_x);
       HYPRE_IJVectorSetObjectType(ij_x, HYPRE_PARCSR);
       HYPRE_IJVectorInitialize(ij_x);
@@ -1509,11 +2093,11 @@ main( hypre_int argc,
    {
       if (myid == 0)
       {
-        hypre_printf("  RHS vector set for solution with unit components\n");
-        hypre_printf("  Initial guess is 0\n");
+         hypre_printf("  RHS vector set for solution with unit components\n");
+         hypre_printf("  Initial guess is 0\n");
       }
 
-/* Temporary use of solution vector */
+      /* Temporary use of solution vector */
       HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, first_local_col, last_local_col, &ij_x);
       HYPRE_IJVectorSetObjectType(ij_x, HYPRE_PARCSR);
       HYPRE_IJVectorInitialize(ij_x);
@@ -1527,7 +2111,7 @@ main( hypre_int argc,
       ierr = HYPRE_IJVectorGetObject( ij_x, &object );
       x = (HYPRE_ParVector) object;
 
-/* RHS */
+      /* RHS */
       HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, first_local_row, last_local_row, &ij_b); 
       HYPRE_IJVectorSetObjectType(ij_b, HYPRE_PARCSR);
       HYPRE_IJVectorInitialize(ij_b);
@@ -1536,7 +2120,7 @@ main( hypre_int argc,
 
       HYPRE_ParCSRMatrixMatvec(1.,parcsr_A,x,0.,b);
 
-/* Initial guess */
+      /* Initial guess */
       values = hypre_CTAlloc(HYPRE_Real, local_num_cols);
       for (i = 0; i < local_num_cols; i++)
          values[i] = 0.;
@@ -1547,11 +2131,11 @@ main( hypre_int argc,
    {
       if (myid == 0)
       {
-        hypre_printf("  RHS vector is 0\n");
-        hypre_printf("  Initial guess has unit components\n");
+         hypre_printf("  RHS vector is 0\n");
+         hypre_printf("  Initial guess has unit components\n");
       }
 
-/* RHS */
+      /* RHS */
       HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, first_local_row, last_local_row, &ij_b);
       HYPRE_IJVectorSetObjectType(ij_b, HYPRE_PARCSR);
       HYPRE_IJVectorInitialize(ij_b);
@@ -1565,7 +2149,7 @@ main( hypre_int argc,
       ierr = HYPRE_IJVectorGetObject( ij_b, &object );
       b = (HYPRE_ParVector) object;
 
-/* Initial guess */
+      /* Initial guess */
       HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, first_local_col, last_local_col, &ij_x);
       HYPRE_IJVectorSetObjectType(ij_x, HYPRE_PARCSR);
       HYPRE_IJVectorInitialize(ij_x);
@@ -1583,23 +2167,27 @@ main( hypre_int argc,
    if ( build_src_type == 0 )
    {
 #if 0
-/* RHS */
+      /* RHS */
       BuildRhsParFromFile(argc, argv, build_src_arg_index, &b);
 #endif
 
       if (myid == 0)
       {
-        hypre_printf("  Source vector read from file %s\n", argv[build_src_arg_index]);
-        hypre_printf("  Initial unknown vector in evolution is 0\n");
+         hypre_printf("  Source vector read from file %s\n", argv[build_src_arg_index]);
+         hypre_printf("  Initial unknown vector in evolution is 0\n");
       }
 
       ierr = HYPRE_IJVectorRead( argv[build_src_arg_index], hypre_MPI_COMM_WORLD, 
                                  HYPRE_PARCSR, &ij_b );
-
+      if (ierr)
+      {
+         hypre_printf("ERROR: Problem reading in the right-hand-side!\n");
+         exit(1);
+      }
       ierr = HYPRE_IJVectorGetObject( ij_b, &object );
       b = (HYPRE_ParVector) object;
 
-/* Initial unknown vector */
+      /* Initial unknown vector */
       HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, first_local_col, last_local_col, &ij_x);
       HYPRE_IJVectorSetObjectType(ij_x, HYPRE_PARCSR);
       HYPRE_IJVectorInitialize(ij_x);
@@ -1619,18 +2207,18 @@ main( hypre_int argc,
       return(-1);
 
 #if 0
-      BuildRhsParFromOneFile2(argc, argv, build_src_arg_index, part_b, &b);
+      BuildRhsParFromOneFile(argc, argv, build_src_arg_index, part_b, &b);
 #endif
    }
    else if ( build_src_type == 2 )
    {
       if (myid == 0)
       {
-        hypre_printf("  Source vector has unit components\n");
-        hypre_printf("  Initial unknown vector is 0\n");
+         hypre_printf("  Source vector has unit components\n");
+         hypre_printf("  Initial unknown vector is 0\n");
       }
 
-/* RHS */
+      /* RHS */
       HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, first_local_row, last_local_row, &ij_b);
       HYPRE_IJVectorSetObjectType(ij_b, HYPRE_PARCSR);
       HYPRE_IJVectorInitialize(ij_b);
@@ -1644,13 +2232,13 @@ main( hypre_int argc,
       ierr = HYPRE_IJVectorGetObject( ij_b, &object );
       b = (HYPRE_ParVector) object;
 
-/* Initial guess */
+      /* Initial guess */
       HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, first_local_col, last_local_col, &ij_x);
       HYPRE_IJVectorSetObjectType(ij_x, HYPRE_PARCSR);
       HYPRE_IJVectorInitialize(ij_x);
 
-/* For backward Euler the previous backward Euler iterate (assumed
-   0 here) is usually used as the initial guess */
+      /* For backward Euler the previous backward Euler iterate (assumed
+         0 here) is usually used as the initial guess */
       values = hypre_CTAlloc(HYPRE_Real, local_num_cols);
       for (i = 0; i < local_num_cols; i++)
          values[i] = 0.;
@@ -1664,11 +2252,11 @@ main( hypre_int argc,
    {
       if (myid == 0)
       {
-        hypre_printf("  Source vector has random components in range 0 - 1\n");
-        hypre_printf("  Initial unknown vector is 0\n");
+         hypre_printf("  Source vector has random components in range 0 - 1\n");
+         hypre_printf("  Initial unknown vector is 0\n");
       }
 
-/* RHS */
+      /* RHS */
       HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, first_local_row, last_local_row, &ij_b);
       HYPRE_IJVectorSetObjectType(ij_b, HYPRE_PARCSR);
       HYPRE_IJVectorInitialize(ij_b);
@@ -1684,13 +2272,13 @@ main( hypre_int argc,
       ierr = HYPRE_IJVectorGetObject( ij_b, &object );
       b = (HYPRE_ParVector) object;
 
-/* Initial guess */
+      /* Initial guess */
       HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, first_local_col, last_local_col, &ij_x);
       HYPRE_IJVectorSetObjectType(ij_x, HYPRE_PARCSR);
       HYPRE_IJVectorInitialize(ij_x);
 
-/* For backward Euler the previous backward Euler iterate (assumed
-   0 here) is usually used as the initial guess */
+      /* For backward Euler the previous backward Euler iterate (assumed
+         0 here) is usually used as the initial guess */
       values = hypre_CTAlloc(HYPRE_Real, local_num_cols);
       for (i = 0; i < local_num_cols; i++)
          values[i] = 0.;
@@ -1704,11 +2292,11 @@ main( hypre_int argc,
    {
       if (myid == 0)
       {
-        hypre_printf("  Source vector is 0 \n");
-        hypre_printf("  Initial unknown vector has random components in range 0 - 1\n");
+         hypre_printf("  Source vector is 0 \n");
+         hypre_printf("  Initial unknown vector has random components in range 0 - 1\n");
       }
 
-/* RHS */
+      /* RHS */
       HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, first_local_row, last_local_row, &ij_b);
       HYPRE_IJVectorSetObjectType(ij_b, HYPRE_PARCSR);
       HYPRE_IJVectorInitialize(ij_b);
@@ -1723,13 +2311,13 @@ main( hypre_int argc,
       ierr = HYPRE_IJVectorGetObject( ij_b, &object );
       b = (HYPRE_ParVector) object;
 
-/* Initial guess */
+      /* Initial guess */
       HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, first_local_col, last_local_col, &ij_x);
       HYPRE_IJVectorSetObjectType(ij_x, HYPRE_PARCSR);
       HYPRE_IJVectorInitialize(ij_x);
 
-/* For backward Euler the previous backward Euler iterate (assumed
-   random in 0 - 1 here) is usually used as the initial guess */
+      /* For backward Euler the previous backward Euler iterate (assumed
+         random in 0 - 1 here) is usually used as the initial guess */
       values = hypre_CTAlloc(HYPRE_Real, local_num_cols);
       hypre_SeedRand(myid);
       for (i = 0; i < local_num_cols; i++)
@@ -1746,9 +2334,6 @@ main( hypre_int argc,
    hypre_FinalizeTiming(time_index);
    hypre_ClearTiming();
    
-   /* HYPRE_IJMatrixPrint(ij_A, "driver.out.A");
-   HYPRE_IJVectorPrint(ij_x, "driver.out.x0"); */
-
    if (num_functions > 1)
    {
       dof_func = NULL;
@@ -1762,26 +2347,8 @@ main( hypre_int argc,
       }
       else
       {
-         local_num_vars = local_num_rows;
-         dof_func = hypre_CTAlloc(HYPRE_Int,local_num_vars);
          if (myid == 0)
-	    hypre_printf (" Number of unknown functions = %d \n", num_functions);
-         rest = first_local_row-((first_local_row/num_functions)*num_functions);
-         indx = num_functions-rest;
-         if (rest == 0) indx = 0;
-         k = num_functions - 1;
-         for (j = indx-1; j > -1; j--)
-	    dof_func[j] = k--;
-         tms = local_num_vars/num_functions;
-         if (tms*num_functions+indx > local_num_vars) tms--;
-         for (j=0; j < tms; j++)
-         {
-	    for (k=0; k < num_functions; k++)
-	       dof_func[indx++] = k;
-         }
-         k = 0;
-         while (indx < local_num_vars)
-	    dof_func[indx++] = k++;
+	    hypre_printf (" Number of functions = %d \n", num_functions);
       }
    }
  
@@ -1794,6 +2361,8 @@ main( hypre_int argc,
       HYPRE_IJMatrixPrint(ij_A, "IJ.out.A");
       HYPRE_IJVectorPrint(ij_b, "IJ.out.b");
       HYPRE_IJVectorPrint(ij_x, "IJ.out.x0");
+
+      /* HYPRE_ParCSRMatrixPrint( parcsr_A, "new_mat.A" );*/
    }
 
    /*-----------------------------------------------------------
@@ -1802,30 +2371,47 @@ main( hypre_int argc,
 
    if (solver_id == 20)
    {
-      dscg_max_its = 1000;
-      pcg_max_its = 200;
       if (myid == 0) hypre_printf("Solver:  AMG\n");
       time_index = hypre_InitializeTiming("AMG_hybrid Setup");
       hypre_BeginTiming(time_index);
 
-      HYPRE_ParCSRHybridCreate(&amg_solver); 
+      HYPRE_ParCSRHybridCreate(&amg_solver);
       HYPRE_ParCSRHybridSetTol(amg_solver, tol);
+      HYPRE_ParCSRHybridSetAbsoluteTol(amg_solver, atol);
       HYPRE_ParCSRHybridSetConvergenceTol(amg_solver, cf_tol);
       HYPRE_ParCSRHybridSetSolverType(amg_solver, solver_type);
       HYPRE_ParCSRHybridSetLogging(amg_solver, ioutdat);
       HYPRE_ParCSRHybridSetPrintLevel(amg_solver, poutdat);
-      HYPRE_ParCSRHybridSetDSCGMaxIter(amg_solver, dscg_max_its );
-      HYPRE_ParCSRHybridSetPCGMaxIter(amg_solver, pcg_max_its ); 
-      HYPRE_ParCSRHybridSetCoarsenType(amg_solver, (hybrid*coarsen_type));
+      HYPRE_ParCSRHybridSetDSCGMaxIter(amg_solver, max_iter);
+      HYPRE_ParCSRHybridSetPCGMaxIter(amg_solver, mg_max_iter);
+      HYPRE_ParCSRHybridSetCoarsenType(amg_solver, coarsen_type);
       HYPRE_ParCSRHybridSetStrongThreshold(amg_solver, strong_threshold);
       HYPRE_ParCSRHybridSetTruncFactor(amg_solver, trunc_factor);
-      HYPRE_ParCSRHybridSetNumGridSweeps(amg_solver, num_grid_sweeps);
-      HYPRE_ParCSRHybridSetGridRelaxType(amg_solver, grid_relax_type);
-      HYPRE_ParCSRHybridSetRelaxWeight(amg_solver, relax_weight);
-      HYPRE_ParCSRHybridSetOmega(amg_solver, omega);
-      HYPRE_ParCSRHybridSetGridRelaxPoints(amg_solver, grid_relax_points);
+      HYPRE_ParCSRHybridSetPMaxElmts(amg_solver, P_max_elmts);
       HYPRE_ParCSRHybridSetMaxLevels(amg_solver, max_levels);
       HYPRE_ParCSRHybridSetMaxRowSum(amg_solver, max_row_sum);
+      HYPRE_ParCSRHybridSetNumSweeps(amg_solver, num_sweeps);
+      if (relax_type > -1) HYPRE_ParCSRHybridSetRelaxType(amg_solver, relax_type);
+      HYPRE_ParCSRHybridSetAggNumLevels(amg_solver, agg_num_levels);
+      HYPRE_ParCSRHybridSetNumPaths(amg_solver, num_paths);
+      HYPRE_ParCSRHybridSetNumFunctions(amg_solver, num_functions);
+      HYPRE_ParCSRHybridSetNodal(amg_solver, nodal);
+      if (relax_down > -1)
+         HYPRE_ParCSRHybridSetCycleRelaxType(amg_solver, relax_down, 1);
+      if (relax_up > -1)
+         HYPRE_ParCSRHybridSetCycleRelaxType(amg_solver, relax_up, 2);
+      if (relax_coarse > -1)
+         HYPRE_ParCSRHybridSetCycleRelaxType(amg_solver, relax_coarse, 3);
+      HYPRE_ParCSRHybridSetRelaxOrder(amg_solver, relax_order);
+      HYPRE_ParCSRHybridSetMaxCoarseSize(amg_solver, coarse_threshold);
+      HYPRE_ParCSRHybridSetMinCoarseSize(amg_solver, min_coarse_size);
+      HYPRE_ParCSRHybridSetSeqThreshold(amg_solver, seq_threshold);
+      HYPRE_ParCSRHybridSetRelaxWt(amg_solver, relax_wt);
+      HYPRE_ParCSRHybridSetOuterWt(amg_solver, outer_wt);
+      if (level_w > -1)
+         HYPRE_ParCSRHybridSetLevelRelaxWt(amg_solver, relax_wt_level, level_w);
+      if (level_ow > -1)
+         HYPRE_ParCSRHybridSetLevelOuterWt(amg_solver, outer_wt_level, level_ow);
   
       HYPRE_ParCSRHybridSetup(amg_solver, parcsr_A, b, x);
 
@@ -1848,7 +2434,7 @@ main( hypre_int argc,
       HYPRE_ParCSRHybridGetPCGNumIterations(amg_solver, &pcg_num_its);
       HYPRE_ParCSRHybridGetDSCGNumIterations(amg_solver, &dscg_num_its);
       HYPRE_ParCSRHybridGetFinalRelativeResidualNorm(amg_solver, 
-	&final_res_norm);
+                                                     &final_res_norm);
 
       if (myid == 0)
       {
@@ -1859,6 +2445,7 @@ main( hypre_int argc,
          hypre_printf("Final Relative Residual Norm = %e\n", final_res_norm);
          hypre_printf("\n");
       }
+
       HYPRE_ParCSRHybridDestroy(amg_solver);
    }
    /*-----------------------------------------------------------
@@ -1875,39 +2462,108 @@ main( hypre_int argc,
       /* BM Aug 25, 2006 */
       HYPRE_BoomerAMGSetCGCIts(amg_solver, cgcits);
       HYPRE_BoomerAMGSetInterpType(amg_solver, interp_type);
+      HYPRE_BoomerAMGSetPostInterpType(amg_solver, post_interp_type);
       HYPRE_BoomerAMGSetNumSamples(amg_solver, gsmg_samples);
-      HYPRE_BoomerAMGSetCoarsenType(amg_solver, (hybrid*coarsen_type));
+      HYPRE_BoomerAMGSetCoarsenType(amg_solver, coarsen_type);
       HYPRE_BoomerAMGSetMeasureType(amg_solver, measure_type);
       HYPRE_BoomerAMGSetTol(amg_solver, tol);
       HYPRE_BoomerAMGSetStrongThreshold(amg_solver, strong_threshold);
+      HYPRE_BoomerAMGSetSeqThreshold(amg_solver, seq_threshold);
+      HYPRE_BoomerAMGSetRedundant(amg_solver, redundant);
+      HYPRE_BoomerAMGSetMaxCoarseSize(amg_solver, coarse_threshold);
+      HYPRE_BoomerAMGSetMinCoarseSize(amg_solver, min_coarse_size);
       HYPRE_BoomerAMGSetTruncFactor(amg_solver, trunc_factor);
       HYPRE_BoomerAMGSetPMaxElmts(amg_solver, P_max_elmts);
-      HYPRE_BoomerAMGSetPostInterpType(amg_solver, post_interp_type);
       HYPRE_BoomerAMGSetJacobiTruncThreshold(amg_solver, jacobi_trunc_threshold);
+      HYPRE_BoomerAMGSetSCommPkgSwitch(amg_solver, S_commpkg_switch);
 /* note: log is written to standard output, not to file */
       HYPRE_BoomerAMGSetPrintLevel(amg_solver, 3);
       HYPRE_BoomerAMGSetPrintFileName(amg_solver, "driver.out.log"); 
       HYPRE_BoomerAMGSetCycleType(amg_solver, cycle_type);
-      HYPRE_BoomerAMGSetNumGridSweeps(amg_solver, num_grid_sweeps);
-      HYPRE_BoomerAMGSetGridRelaxType(amg_solver, grid_relax_type);
-      HYPRE_BoomerAMGSetRelaxWeight(amg_solver, relax_weight);
-      HYPRE_BoomerAMGSetOmega(amg_solver, omega);
+      HYPRE_BoomerAMGSetNumSweeps(amg_solver, num_sweeps);
+      HYPRE_BoomerAMGSetISType(amg_solver, IS_type);
+      HYPRE_BoomerAMGSetNumCRRelaxSteps(amg_solver, num_CR_relax_steps);
+      HYPRE_BoomerAMGSetCRRate(amg_solver, CR_rate);
+      HYPRE_BoomerAMGSetCRStrongTh(amg_solver, CR_strong_th);
+      HYPRE_BoomerAMGSetCRUseCG(amg_solver, CR_use_CG);
+      if (relax_type > -1) HYPRE_BoomerAMGSetRelaxType(amg_solver, relax_type);
+      if (relax_down > -1)
+         HYPRE_BoomerAMGSetCycleRelaxType(amg_solver, relax_down, 1);
+      if (relax_up > -1)
+         HYPRE_BoomerAMGSetCycleRelaxType(amg_solver, relax_up, 2);
+      if (relax_coarse > -1)
+         HYPRE_BoomerAMGSetCycleRelaxType(amg_solver, relax_coarse, 3);
+      HYPRE_BoomerAMGSetAddRelaxType(amg_solver, add_relax_type);
+      HYPRE_BoomerAMGSetAddRelaxWt(amg_solver, add_relax_wt);
+      HYPRE_BoomerAMGSetChebyOrder(amg_solver, cheby_order);
+      HYPRE_BoomerAMGSetChebyFraction(amg_solver, cheby_fraction);
+      HYPRE_BoomerAMGSetRelaxOrder(amg_solver, relax_order);
+      HYPRE_BoomerAMGSetRelaxWt(amg_solver, relax_wt);
+      HYPRE_BoomerAMGSetOuterWt(amg_solver, outer_wt);
+      HYPRE_BoomerAMGSetMaxLevels(amg_solver, max_levels);
+      if (level_w > -1)
+         HYPRE_BoomerAMGSetLevelRelaxWt(amg_solver, relax_wt_level, level_w);
+      if (level_ow > -1)
+         HYPRE_BoomerAMGSetLevelOuterWt(amg_solver, outer_wt_level, level_ow);
       HYPRE_BoomerAMGSetSmoothType(amg_solver, smooth_type);
       HYPRE_BoomerAMGSetSmoothNumSweeps(amg_solver, smooth_num_sweeps);
       HYPRE_BoomerAMGSetSmoothNumLevels(amg_solver, smooth_num_levels);
-      HYPRE_BoomerAMGSetGridRelaxPoints(amg_solver, grid_relax_points);
-      HYPRE_BoomerAMGSetMaxLevels(amg_solver, max_levels);
       HYPRE_BoomerAMGSetMaxRowSum(amg_solver, max_row_sum);
       HYPRE_BoomerAMGSetDebugFlag(amg_solver, debug_flag);
       HYPRE_BoomerAMGSetVariant(amg_solver, variant);
       HYPRE_BoomerAMGSetOverlap(amg_solver, overlap);
       HYPRE_BoomerAMGSetDomainType(amg_solver, domain_type);
+      HYPRE_BoomerAMGSetSchwarzUseNonSymm(amg_solver, use_nonsymm_schwarz);
+     
       HYPRE_BoomerAMGSetSchwarzRlxWeight(amg_solver, schwarz_rlx_weight);
+      if (eu_level < 0) eu_level = 0;
+      HYPRE_BoomerAMGSetEuLevel(amg_solver, eu_level);
+      HYPRE_BoomerAMGSetEuBJ(amg_solver, eu_bj);
+      HYPRE_BoomerAMGSetEuSparseA(amg_solver, eu_sparse_A);
       HYPRE_BoomerAMGSetNumFunctions(amg_solver, num_functions);
-      HYPRE_BoomerAMGSetNumPaths(amg_solver, num_paths);
       HYPRE_BoomerAMGSetAggNumLevels(amg_solver, agg_num_levels);
+      HYPRE_BoomerAMGSetAggInterpType(amg_solver, agg_interp_type);
+      HYPRE_BoomerAMGSetAggTruncFactor(amg_solver, agg_trunc_factor);
+      HYPRE_BoomerAMGSetAggP12TruncFactor(amg_solver, agg_P12_trunc_factor);
+      HYPRE_BoomerAMGSetAggPMaxElmts(amg_solver, agg_P_max_elmts);
+      HYPRE_BoomerAMGSetAggP12MaxElmts(amg_solver, agg_P12_max_elmts);
+      HYPRE_BoomerAMGSetNumPaths(amg_solver, num_paths);
+      HYPRE_BoomerAMGSetNodal(amg_solver, nodal);
+      HYPRE_BoomerAMGSetNodalDiag(amg_solver, nodal_diag);
+      HYPRE_BoomerAMGSetCycleNumSweeps(amg_solver, ns_coarse, 3);
       if (num_functions > 1)
 	 HYPRE_BoomerAMGSetDofFunc(amg_solver, dof_func);
+      HYPRE_BoomerAMGSetAdditive(amg_solver, additive);
+      HYPRE_BoomerAMGSetMultAdditive(amg_solver, mult_add);
+      HYPRE_BoomerAMGSetSimple(amg_solver, simple);
+      HYPRE_BoomerAMGSetMultAddPMaxElmts(amg_solver, add_P_max_elmts);
+      HYPRE_BoomerAMGSetMultAddTruncFactor(amg_solver, add_trunc_factor);
+
+      HYPRE_BoomerAMGSetMaxIter(amg_solver, mg_max_iter);
+      HYPRE_BoomerAMGSetRAP2(amg_solver, rap2);
+      HYPRE_BoomerAMGSetKeepTranspose(amg_solver, keepTranspose);
+      /*HYPRE_BoomerAMGSetNonGalerkTol(amg_solver, nongalerk_num_tol, nongalerk_tol);*/
+      if (nongalerk_tol)
+      {
+         HYPRE_BoomerAMGSetNonGalerkinTol(amg_solver, nongalerk_tol[nongalerk_num_tol-1]);
+         for (i=0; i < nongalerk_num_tol-1; i++)
+            HYPRE_BoomerAMGSetLevelNonGalerkinTol(amg_solver, nongalerk_tol[i], i);
+      }
+      if (build_rbm)
+      {
+         HYPRE_BoomerAMGSetInterpVectors(amg_solver, num_interp_vecs, interp_vecs);
+         HYPRE_BoomerAMGSetInterpVecVariant(amg_solver, interp_vec_variant);
+         HYPRE_BoomerAMGSetInterpVecQMax(amg_solver, Q_max);
+         HYPRE_BoomerAMGSetInterpVecAbsQTrunc(amg_solver, Q_trunc);
+      }
+
+      /* BM Oct 23, 2006 */
+      if (plot_grids) {
+         HYPRE_BoomerAMGSetPlotGrids (amg_solver, 1);
+         HYPRE_BoomerAMGSetPlotFileName (amg_solver, plot_file_name);
+         HYPRE_BoomerAMGSetCoordDim (amg_solver, coord_dim);
+         HYPRE_BoomerAMGSetCoordinates (amg_solver, coordinates);
+      }
 
       HYPRE_BoomerAMGSetup(amg_solver, parcsr_A, b, x);
 
@@ -1955,79 +2611,95 @@ main( hypre_int argc,
    {
       /* reset some smoother parameters */
  
-      /* fine grid */
-      num_grid_sweeps[0] = num_sweep;
-      grid_relax_type[0] = relax_default;
-      hypre_TFree (grid_relax_points[0]);
-      grid_relax_points[0] = hypre_CTAlloc(HYPRE_Int, num_sweep);
-      for (i=0; i<num_sweep; i++)
-         grid_relax_points[0][i] = 0;
- 
-      /* down cycle */
-      num_grid_sweeps[1] = num_sweep;
-      grid_relax_type[1] = relax_default;
-      hypre_TFree (grid_relax_points[1]);
-      grid_relax_points[1] = hypre_CTAlloc(HYPRE_Int, num_sweep);
-      for (i=0; i<num_sweep; i++)
-         grid_relax_points[1][i] = 0;
- 
-      /* up cycle */
-      num_grid_sweeps[2] = num_sweep;
-      grid_relax_type[2] = relax_default;
-      hypre_TFree (grid_relax_points[2]);
-      grid_relax_points[2] = hypre_CTAlloc(HYPRE_Int, num_sweep);
-      for (i=0; i<num_sweep; i++)
-         grid_relax_points[2][i] = 0;
- 
-      /* coarsest grid */
-      num_grid_sweeps[3] = 1;
-      grid_relax_type[3] = 9;
-      hypre_TFree (grid_relax_points[3]);
-      grid_relax_points[3] = hypre_CTAlloc(HYPRE_Int, 1);
-      grid_relax_points[3][0] = 0;
+      relax_order = 0;
  
       if (myid == 0) hypre_printf("Solver:  GSMG\n");
       time_index = hypre_InitializeTiming("BoomerAMG Setup");
       hypre_BeginTiming(time_index);
  
       HYPRE_BoomerAMGCreate(&amg_solver);
+      HYPRE_BoomerAMGSetGSMG(amg_solver, 4); /* specify GSMG */
       /* BM Aug 25, 2006 */
       HYPRE_BoomerAMGSetCGCIts(amg_solver, cgcits);
-      HYPRE_BoomerAMGSetGSMG(amg_solver, 4); /* specify GSMG */
       HYPRE_BoomerAMGSetInterpType(amg_solver, interp_type);
+      HYPRE_BoomerAMGSetPostInterpType(amg_solver, post_interp_type);
       HYPRE_BoomerAMGSetNumSamples(amg_solver, gsmg_samples);
-      HYPRE_BoomerAMGSetCoarsenType(amg_solver, (hybrid*coarsen_type));
+      HYPRE_BoomerAMGSetCoarsenType(amg_solver, coarsen_type);
       HYPRE_BoomerAMGSetMeasureType(amg_solver, measure_type);
       HYPRE_BoomerAMGSetTol(amg_solver, tol);
       HYPRE_BoomerAMGSetStrongThreshold(amg_solver, strong_threshold);
+      HYPRE_BoomerAMGSetSeqThreshold(amg_solver, seq_threshold);
+      HYPRE_BoomerAMGSetRedundant(amg_solver, redundant);
+      HYPRE_BoomerAMGSetMaxCoarseSize(amg_solver, coarse_threshold);
+      HYPRE_BoomerAMGSetMinCoarseSize(amg_solver, min_coarse_size);
       HYPRE_BoomerAMGSetTruncFactor(amg_solver, trunc_factor);
       HYPRE_BoomerAMGSetPMaxElmts(amg_solver, P_max_elmts);
-      HYPRE_BoomerAMGSetPostInterpType(amg_solver, post_interp_type);
       HYPRE_BoomerAMGSetJacobiTruncThreshold(amg_solver, jacobi_trunc_threshold);
-/* note: log is written to standard output, not to file */
+      HYPRE_BoomerAMGSetSCommPkgSwitch(amg_solver, S_commpkg_switch);
+      /* note: log is written to standard output, not to file */
       HYPRE_BoomerAMGSetPrintLevel(amg_solver, 3);
       HYPRE_BoomerAMGSetPrintFileName(amg_solver, "driver.out.log");
+      HYPRE_BoomerAMGSetMaxIter(amg_solver, mg_max_iter);
       HYPRE_BoomerAMGSetCycleType(amg_solver, cycle_type);
-      HYPRE_BoomerAMGSetNumGridSweeps(amg_solver, num_grid_sweeps);
-      HYPRE_BoomerAMGSetGridRelaxType(amg_solver, grid_relax_type);
-      HYPRE_BoomerAMGSetRelaxWeight(amg_solver, relax_weight);
-      HYPRE_BoomerAMGSetOmega(amg_solver, omega);
+      HYPRE_BoomerAMGSetNumSweeps(amg_solver, num_sweeps);
+      if (relax_type > -1) HYPRE_BoomerAMGSetRelaxType(amg_solver, relax_type);
+      if (relax_down > -1)
+         HYPRE_BoomerAMGSetCycleRelaxType(amg_solver, relax_down, 1);
+      if (relax_up > -1)
+         HYPRE_BoomerAMGSetCycleRelaxType(amg_solver, relax_up, 2);
+      if (relax_coarse > -1)
+         HYPRE_BoomerAMGSetCycleRelaxType(amg_solver, relax_coarse, 3);
+      HYPRE_BoomerAMGSetAddRelaxType(amg_solver, add_relax_type);
+      HYPRE_BoomerAMGSetAddRelaxWt(amg_solver, add_relax_wt);
+      HYPRE_BoomerAMGSetChebyOrder(amg_solver, cheby_order);
+      HYPRE_BoomerAMGSetChebyFraction(amg_solver, cheby_fraction);
+      HYPRE_BoomerAMGSetRelaxOrder(amg_solver, relax_order);
+      HYPRE_BoomerAMGSetRelaxWt(amg_solver, relax_wt);
+      HYPRE_BoomerAMGSetOuterWt(amg_solver, outer_wt);
+      if (level_w > -1)
+         HYPRE_BoomerAMGSetLevelRelaxWt(amg_solver, relax_wt_level, level_w);
+      if (level_ow > -1)
+         HYPRE_BoomerAMGSetLevelOuterWt(amg_solver, outer_wt_level, level_ow);
       HYPRE_BoomerAMGSetSmoothType(amg_solver, smooth_type);
       HYPRE_BoomerAMGSetSmoothNumSweeps(amg_solver, smooth_num_sweeps);
       HYPRE_BoomerAMGSetSmoothNumLevels(amg_solver, smooth_num_levels);
-      HYPRE_BoomerAMGSetGridRelaxPoints(amg_solver, grid_relax_points);
       HYPRE_BoomerAMGSetMaxLevels(amg_solver, max_levels);
       HYPRE_BoomerAMGSetMaxRowSum(amg_solver, max_row_sum);
       HYPRE_BoomerAMGSetDebugFlag(amg_solver, debug_flag);
       HYPRE_BoomerAMGSetVariant(amg_solver, variant);
       HYPRE_BoomerAMGSetOverlap(amg_solver, overlap);
       HYPRE_BoomerAMGSetDomainType(amg_solver, domain_type);
+      HYPRE_BoomerAMGSetSchwarzUseNonSymm(amg_solver, use_nonsymm_schwarz);
       HYPRE_BoomerAMGSetSchwarzRlxWeight(amg_solver, schwarz_rlx_weight);
+      if (eu_level < 0) eu_level = 0;
+      HYPRE_BoomerAMGSetEuLevel(amg_solver, eu_level);
+      HYPRE_BoomerAMGSetEuBJ(amg_solver, eu_bj);
+      HYPRE_BoomerAMGSetEuSparseA(amg_solver, eu_sparse_A);
       HYPRE_BoomerAMGSetNumFunctions(amg_solver, num_functions);
-      HYPRE_BoomerAMGSetNumPaths(amg_solver, num_paths);
       HYPRE_BoomerAMGSetAggNumLevels(amg_solver, agg_num_levels);
+      HYPRE_BoomerAMGSetAggInterpType(amg_solver, agg_interp_type);
+      HYPRE_BoomerAMGSetAggTruncFactor(amg_solver, agg_trunc_factor);
+      HYPRE_BoomerAMGSetAggP12TruncFactor(amg_solver, agg_P12_trunc_factor);
+      HYPRE_BoomerAMGSetAggPMaxElmts(amg_solver, agg_P_max_elmts);
+      HYPRE_BoomerAMGSetAggP12MaxElmts(amg_solver, agg_P12_max_elmts);
+      HYPRE_BoomerAMGSetNumPaths(amg_solver, num_paths);
+      HYPRE_BoomerAMGSetNodal(amg_solver, nodal);
+      HYPRE_BoomerAMGSetNodalDiag(amg_solver, nodal_diag);
       if (num_functions > 1)
          HYPRE_BoomerAMGSetDofFunc(amg_solver, dof_func);
+      HYPRE_BoomerAMGSetAdditive(amg_solver, additive);
+      HYPRE_BoomerAMGSetMultAdditive(amg_solver, mult_add);
+      HYPRE_BoomerAMGSetSimple(amg_solver, simple);
+      HYPRE_BoomerAMGSetMultAddPMaxElmts(amg_solver, add_P_max_elmts);
+      HYPRE_BoomerAMGSetMultAddTruncFactor(amg_solver, add_trunc_factor);
+      HYPRE_BoomerAMGSetRAP2(amg_solver, rap2);
+      HYPRE_BoomerAMGSetKeepTranspose(amg_solver, keepTranspose);
+      if (nongalerk_tol)
+      {
+         HYPRE_BoomerAMGSetNonGalerkinTol(amg_solver, nongalerk_tol[nongalerk_num_tol-1]);
+         for (i=0; i < nongalerk_num_tol-1; i++)
+            HYPRE_BoomerAMGSetLevelNonGalerkinTol(amg_solver, nongalerk_tol[i], i);
+      }
  
       HYPRE_BoomerAMGSetup(amg_solver, parcsr_A, b, x);
  
@@ -2087,20 +2759,21 @@ main( hypre_int argc,
     *-----------------------------------------------------------*/
 
    /* begin lobpcg */
-   if ( !lobpcgFlag && ( solver_id == 1 || solver_id == 2 || solver_id == 8 || 
-			 solver_id == 12 || solver_id == 14 || solver_id == 43 ) )
+   if (!lobpcgFlag && (solver_id == 1 || solver_id == 2 || solver_id == 8 || 
+       solver_id == 12 || solver_id == 14 || solver_id == 43))
    /*end lobpcg */
    {
       time_index = hypre_InitializeTiming("PCG Setup");
       hypre_BeginTiming(time_index);
  
       HYPRE_ParCSRPCGCreate(hypre_MPI_COMM_WORLD, &pcg_solver);
-      HYPRE_PCGSetMaxIter(pcg_solver, 1000);
+      HYPRE_PCGSetMaxIter(pcg_solver, max_iter);
       HYPRE_PCGSetTol(pcg_solver, tol);
-      HYPRE_PCGSetTwoNorm(pcg_solver, two_norm);
-      HYPRE_PCGSetRelChange(pcg_solver, 0);
+      HYPRE_PCGSetTwoNorm(pcg_solver, 1);
+      HYPRE_PCGSetRelChange(pcg_solver, rel_change);
       HYPRE_PCGSetPrintLevel(pcg_solver, ioutdat);
- 
+      HYPRE_PCGSetAbsoluteTol(pcg_solver, atol);
+
       if (solver_id == 1)
       {
          /* use BoomerAMG as preconditioner */
@@ -2109,42 +2782,101 @@ main( hypre_int argc,
          /* BM Aug 25, 2006 */
          HYPRE_BoomerAMGSetCGCIts(pcg_precond, cgcits);
          HYPRE_BoomerAMGSetInterpType(pcg_precond, interp_type);
+         HYPRE_BoomerAMGSetPostInterpType(pcg_precond, post_interp_type);
          HYPRE_BoomerAMGSetNumSamples(pcg_precond, gsmg_samples);
          HYPRE_BoomerAMGSetTol(pcg_precond, pc_tol);
-         HYPRE_BoomerAMGSetCoarsenType(pcg_precond, (hybrid*coarsen_type));
+         HYPRE_BoomerAMGSetCoarsenType(pcg_precond, coarsen_type);
          HYPRE_BoomerAMGSetMeasureType(pcg_precond, measure_type);
          HYPRE_BoomerAMGSetStrongThreshold(pcg_precond, strong_threshold);
+         HYPRE_BoomerAMGSetSeqThreshold(pcg_precond, seq_threshold);
+         HYPRE_BoomerAMGSetRedundant(pcg_precond, redundant);
+         HYPRE_BoomerAMGSetMaxCoarseSize(pcg_precond, coarse_threshold);
+         HYPRE_BoomerAMGSetMinCoarseSize(pcg_precond, min_coarse_size);
          HYPRE_BoomerAMGSetTruncFactor(pcg_precond, trunc_factor);
          HYPRE_BoomerAMGSetPMaxElmts(pcg_precond, P_max_elmts);
-         HYPRE_BoomerAMGSetPostInterpType(pcg_precond, post_interp_type);
          HYPRE_BoomerAMGSetJacobiTruncThreshold(pcg_precond, jacobi_trunc_threshold);
+         HYPRE_BoomerAMGSetSCommPkgSwitch(pcg_precond, S_commpkg_switch);
          HYPRE_BoomerAMGSetPrintLevel(pcg_precond, poutdat);
          HYPRE_BoomerAMGSetPrintFileName(pcg_precond, "driver.out.log");
          HYPRE_BoomerAMGSetMaxIter(pcg_precond, 1);
          HYPRE_BoomerAMGSetCycleType(pcg_precond, cycle_type);
-         HYPRE_BoomerAMGSetNumGridSweeps(pcg_precond, num_grid_sweeps);
-         HYPRE_BoomerAMGSetGridRelaxType(pcg_precond, grid_relax_type);
-         HYPRE_BoomerAMGSetRelaxWeight(pcg_precond, relax_weight);
-         HYPRE_BoomerAMGSetOmega(pcg_precond, omega);
+         HYPRE_BoomerAMGSetNumSweeps(pcg_precond, num_sweeps);
+         HYPRE_BoomerAMGSetISType(pcg_precond, IS_type);
+         HYPRE_BoomerAMGSetNumCRRelaxSteps(pcg_precond, num_CR_relax_steps);
+         HYPRE_BoomerAMGSetCRRate(pcg_precond, CR_rate);
+         HYPRE_BoomerAMGSetCRStrongTh(pcg_precond, CR_strong_th);
+         HYPRE_BoomerAMGSetCRUseCG(pcg_precond, CR_use_CG);
+         if (relax_type > -1) HYPRE_BoomerAMGSetRelaxType(pcg_precond, relax_type);
+         if (relax_down > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_down, 1);
+         if (relax_up > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_up, 2);
+         if (relax_coarse > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_coarse, 3);
+         HYPRE_BoomerAMGSetAddRelaxType(pcg_precond, add_relax_type);
+         HYPRE_BoomerAMGSetAddRelaxWt(pcg_precond, add_relax_wt);
+         HYPRE_BoomerAMGSetChebyOrder(pcg_precond, cheby_order);
+         HYPRE_BoomerAMGSetChebyFraction(pcg_precond, cheby_fraction);
+         HYPRE_BoomerAMGSetRelaxOrder(pcg_precond, relax_order);
+         HYPRE_BoomerAMGSetRelaxWt(pcg_precond, relax_wt);
+         HYPRE_BoomerAMGSetOuterWt(pcg_precond, outer_wt);
+         if (level_w > -1)
+            HYPRE_BoomerAMGSetLevelRelaxWt(pcg_precond, relax_wt_level,level_w);
+         if (level_ow > -1)
+            HYPRE_BoomerAMGSetLevelOuterWt(pcg_precond,outer_wt_level,level_ow);
          HYPRE_BoomerAMGSetSmoothType(pcg_precond, smooth_type);
          HYPRE_BoomerAMGSetSmoothNumLevels(pcg_precond, smooth_num_levels);
          HYPRE_BoomerAMGSetSmoothNumSweeps(pcg_precond, smooth_num_sweeps);
-         HYPRE_BoomerAMGSetGridRelaxPoints(pcg_precond, grid_relax_points);
          HYPRE_BoomerAMGSetMaxLevels(pcg_precond, max_levels);
          HYPRE_BoomerAMGSetMaxRowSum(pcg_precond, max_row_sum);
+         HYPRE_BoomerAMGSetDebugFlag(pcg_precond, debug_flag);
          HYPRE_BoomerAMGSetNumFunctions(pcg_precond, num_functions);
-         HYPRE_BoomerAMGSetNumPaths(pcg_precond, num_paths);
          HYPRE_BoomerAMGSetAggNumLevels(pcg_precond, agg_num_levels);
+         HYPRE_BoomerAMGSetAggInterpType(pcg_precond, agg_interp_type);
+         HYPRE_BoomerAMGSetAggTruncFactor(pcg_precond, agg_trunc_factor);
+         HYPRE_BoomerAMGSetAggP12TruncFactor(pcg_precond, agg_P12_trunc_factor);
+         HYPRE_BoomerAMGSetAggPMaxElmts(pcg_precond, agg_P_max_elmts);
+         HYPRE_BoomerAMGSetAggP12MaxElmts(pcg_precond, agg_P12_max_elmts);
+         HYPRE_BoomerAMGSetNumPaths(pcg_precond, num_paths);
+         HYPRE_BoomerAMGSetNodal(pcg_precond, nodal);
+         HYPRE_BoomerAMGSetNodalDiag(pcg_precond, nodal_diag);
          HYPRE_BoomerAMGSetVariant(pcg_precond, variant);
          HYPRE_BoomerAMGSetOverlap(pcg_precond, overlap);
          HYPRE_BoomerAMGSetDomainType(pcg_precond, domain_type);
+         HYPRE_BoomerAMGSetSchwarzUseNonSymm(pcg_precond, use_nonsymm_schwarz);
          HYPRE_BoomerAMGSetSchwarzRlxWeight(pcg_precond, schwarz_rlx_weight);
+         if (eu_level < 0) eu_level = 0;
+         HYPRE_BoomerAMGSetEuLevel(pcg_precond, eu_level);
+         HYPRE_BoomerAMGSetEuBJ(pcg_precond, eu_bj);
+         HYPRE_BoomerAMGSetEuSparseA(pcg_precond, eu_sparse_A);
+         HYPRE_BoomerAMGSetCycleNumSweeps(pcg_precond, ns_coarse, 3);
          if (num_functions > 1)
             HYPRE_BoomerAMGSetDofFunc(pcg_precond, dof_func);
+         HYPRE_BoomerAMGSetAdditive(pcg_precond, additive);
+         HYPRE_BoomerAMGSetMultAdditive(pcg_precond, mult_add);
+         HYPRE_BoomerAMGSetSimple(pcg_precond, simple);
+         HYPRE_BoomerAMGSetMultAddPMaxElmts(pcg_precond, add_P_max_elmts);
+         HYPRE_BoomerAMGSetMultAddTruncFactor(pcg_precond, add_trunc_factor);
+         HYPRE_BoomerAMGSetRAP2(pcg_precond, rap2);
+         HYPRE_BoomerAMGSetKeepTranspose(pcg_precond, keepTranspose);
+         if (nongalerk_tol)
+         {
+            HYPRE_BoomerAMGSetNonGalerkinTol(pcg_precond, nongalerk_tol[nongalerk_num_tol-1]);
+            for (i=0; i < nongalerk_num_tol-1; i++)
+               HYPRE_BoomerAMGSetLevelNonGalerkinTol(pcg_precond, nongalerk_tol[i], i);
+         }
+         if (build_rbm)
+         {
+            HYPRE_BoomerAMGSetInterpVectors(pcg_precond, num_interp_vecs, interp_vecs);
+            HYPRE_BoomerAMGSetInterpVecVariant(pcg_precond, interp_vec_variant);
+            HYPRE_BoomerAMGSetInterpVecQMax(pcg_precond, Q_max);
+            HYPRE_BoomerAMGSetInterpVecAbsQTrunc(pcg_precond, Q_trunc);
+         }
+         HYPRE_PCGSetMaxIter(pcg_solver, mg_max_iter);
          HYPRE_PCGSetPrecond(pcg_solver,
                              (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
                              (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
-                                   pcg_precond);
+                             pcg_precond);
       }
       else if (solver_id == 2)
       {
@@ -2183,7 +2915,7 @@ main( hypre_int argc,
 	 HYPRE_SchwarzSetOverlap(pcg_precond, overlap);
 	 HYPRE_SchwarzSetDomainType(pcg_precond, domain_type);
          HYPRE_SchwarzSetRelaxWeight(pcg_precond, schwarz_rlx_weight);
-
+         HYPRE_SchwarzSetNonSymm(pcg_precond, use_nonsymm_schwarz);
          HYPRE_PCGSetPrecond(pcg_solver,
                              (HYPRE_PtrToSolverFcn) HYPRE_SchwarzSolve,
                              (HYPRE_PtrToSolverFcn) HYPRE_SchwarzSetup,
@@ -2196,59 +2928,56 @@ main( hypre_int argc,
          /* reset some smoother parameters */
     
          /* fine grid */
-         num_grid_sweeps[0] = num_sweep;
-         grid_relax_type[0] = relax_default;
-         hypre_TFree (grid_relax_points[0]);
-         grid_relax_points[0] = hypre_CTAlloc(HYPRE_Int, num_sweep);
-         for (i=0; i<num_sweep; i++)
-            grid_relax_points[0][i] = 0;
-    
-         /* down cycle */
-         num_grid_sweeps[1] = num_sweep;
-         grid_relax_type[1] = relax_default;
-         hypre_TFree (grid_relax_points[1]);
-         grid_relax_points[1] = hypre_CTAlloc(HYPRE_Int, num_sweep);
-         for (i=0; i<num_sweep; i++)
-            grid_relax_points[1][i] = 0;
-    
-         /* up cycle */
-         num_grid_sweeps[2] = num_sweep;
-         grid_relax_type[2] = relax_default;
-         hypre_TFree (grid_relax_points[2]);
-         grid_relax_points[2] = hypre_CTAlloc(HYPRE_Int, num_sweep);
-         for (i=0; i<num_sweep; i++)
-            grid_relax_points[2][i] = 0;
-    
-         /* coarsest grid */
-         num_grid_sweeps[3] = 1;
-         grid_relax_type[3] = 9;
-         hypre_TFree (grid_relax_points[3]);
-         grid_relax_points[3] = hypre_CTAlloc(HYPRE_Int, 1);
-         grid_relax_points[3][0] = 0;
+         relax_order = 0;
  
          if (myid == 0) hypre_printf("Solver: GSMG-PCG\n");
          HYPRE_BoomerAMGCreate(&pcg_precond); 
+         HYPRE_BoomerAMGSetGSMG(pcg_precond, 4); 
          /* BM Aug 25, 2006 */
          HYPRE_BoomerAMGSetCGCIts(pcg_precond, cgcits);
-         HYPRE_BoomerAMGSetGSMG(pcg_precond, 4); 
          HYPRE_BoomerAMGSetInterpType(pcg_precond, interp_type);
+         HYPRE_BoomerAMGSetPostInterpType(pcg_precond, post_interp_type);
          HYPRE_BoomerAMGSetNumSamples(pcg_precond, gsmg_samples);
          HYPRE_BoomerAMGSetTol(pcg_precond, pc_tol);
-         HYPRE_BoomerAMGSetCoarsenType(pcg_precond, (hybrid*coarsen_type));
+         HYPRE_BoomerAMGSetCoarsenType(pcg_precond, coarsen_type);
          HYPRE_BoomerAMGSetMeasureType(pcg_precond, measure_type);
          HYPRE_BoomerAMGSetStrongThreshold(pcg_precond, strong_threshold);
+         HYPRE_BoomerAMGSetSeqThreshold(pcg_precond, seq_threshold);
+         HYPRE_BoomerAMGSetRedundant(pcg_precond, redundant);
+         HYPRE_BoomerAMGSetMaxCoarseSize(pcg_precond, coarse_threshold);
+         HYPRE_BoomerAMGSetMinCoarseSize(pcg_precond, min_coarse_size);
          HYPRE_BoomerAMGSetTruncFactor(pcg_precond, trunc_factor);
          HYPRE_BoomerAMGSetPMaxElmts(pcg_precond, P_max_elmts);
-         HYPRE_BoomerAMGSetPostInterpType(pcg_precond, post_interp_type);
          HYPRE_BoomerAMGSetJacobiTruncThreshold(pcg_precond, jacobi_trunc_threshold);
+         HYPRE_BoomerAMGSetSCommPkgSwitch(pcg_precond, S_commpkg_switch);
          HYPRE_BoomerAMGSetPrintLevel(pcg_precond, poutdat);
          HYPRE_BoomerAMGSetPrintFileName(pcg_precond, "driver.out.log");
          HYPRE_BoomerAMGSetMaxIter(pcg_precond, 1);
          HYPRE_BoomerAMGSetCycleType(pcg_precond, cycle_type);
-         HYPRE_BoomerAMGSetNumGridSweeps(pcg_precond, num_grid_sweeps);
-         HYPRE_BoomerAMGSetGridRelaxType(pcg_precond, grid_relax_type);
-         HYPRE_BoomerAMGSetRelaxWeight(pcg_precond, relax_weight);
-         HYPRE_BoomerAMGSetOmega(pcg_precond, omega);
+         HYPRE_BoomerAMGSetNumSweeps(pcg_precond, num_sweeps);
+         HYPRE_BoomerAMGSetISType(pcg_precond, IS_type);
+         HYPRE_BoomerAMGSetNumCRRelaxSteps(pcg_precond, num_CR_relax_steps);
+         HYPRE_BoomerAMGSetCRRate(pcg_precond, CR_rate);
+         HYPRE_BoomerAMGSetCRStrongTh(pcg_precond, CR_strong_th);
+         HYPRE_BoomerAMGSetCRUseCG(pcg_precond, CR_use_CG);
+         if (relax_type > -1) HYPRE_BoomerAMGSetRelaxType(pcg_precond, relax_type);
+         if (relax_down > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_down, 1);
+         if (relax_up > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_up, 2);
+         if (relax_coarse > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_coarse, 3);
+         HYPRE_BoomerAMGSetAddRelaxType(pcg_precond, add_relax_type);
+         HYPRE_BoomerAMGSetAddRelaxWt(pcg_precond, add_relax_wt);
+         HYPRE_BoomerAMGSetRelaxOrder(pcg_precond, relax_order);
+         HYPRE_BoomerAMGSetChebyOrder(pcg_precond, cheby_order);
+         HYPRE_BoomerAMGSetChebyFraction(pcg_precond, cheby_fraction);
+         HYPRE_BoomerAMGSetRelaxWt(pcg_precond, relax_wt);
+         HYPRE_BoomerAMGSetOuterWt(pcg_precond, outer_wt);
+         if (level_w > -1)
+            HYPRE_BoomerAMGSetLevelRelaxWt(pcg_precond, relax_wt_level,level_w);
+         if (level_ow > -1)
+            HYPRE_BoomerAMGSetLevelOuterWt(pcg_precond,outer_wt_level,level_ow);
          HYPRE_BoomerAMGSetSmoothType(pcg_precond, smooth_type);
          HYPRE_BoomerAMGSetSmoothNumLevels(pcg_precond, smooth_num_levels);
          HYPRE_BoomerAMGSetSmoothNumSweeps(pcg_precond, smooth_num_sweeps);
@@ -2256,18 +2985,44 @@ main( hypre_int argc,
          HYPRE_BoomerAMGSetOverlap(pcg_precond, overlap);
          HYPRE_BoomerAMGSetDomainType(pcg_precond, domain_type);
          HYPRE_BoomerAMGSetSchwarzRlxWeight(pcg_precond, schwarz_rlx_weight);
-         HYPRE_BoomerAMGSetGridRelaxPoints(pcg_precond, grid_relax_points);
+         if (eu_level < 0) eu_level = 0;
+         HYPRE_BoomerAMGSetEuLevel(pcg_precond, eu_level);
+         HYPRE_BoomerAMGSetEuBJ(pcg_precond, eu_bj);
+         HYPRE_BoomerAMGSetEuSparseA(pcg_precond, eu_sparse_A);
          HYPRE_BoomerAMGSetMaxLevels(pcg_precond, max_levels);
          HYPRE_BoomerAMGSetMaxRowSum(pcg_precond, max_row_sum);
+         HYPRE_BoomerAMGSetDebugFlag(pcg_precond, debug_flag);
          HYPRE_BoomerAMGSetNumFunctions(pcg_precond, num_functions);
-         HYPRE_BoomerAMGSetNumPaths(pcg_precond, num_paths);
          HYPRE_BoomerAMGSetAggNumLevels(pcg_precond, agg_num_levels);
+         HYPRE_BoomerAMGSetAggInterpType(pcg_precond, agg_interp_type);
+         HYPRE_BoomerAMGSetAggTruncFactor(pcg_precond, agg_trunc_factor);
+         HYPRE_BoomerAMGSetAggP12TruncFactor(pcg_precond, agg_P12_trunc_factor);
+         HYPRE_BoomerAMGSetAggPMaxElmts(pcg_precond, agg_P_max_elmts);
+         HYPRE_BoomerAMGSetAggP12MaxElmts(pcg_precond, agg_P12_max_elmts);
+         HYPRE_BoomerAMGSetNumPaths(pcg_precond, num_paths);
+         HYPRE_BoomerAMGSetNodal(pcg_precond, nodal);
+         HYPRE_BoomerAMGSetNodalDiag(pcg_precond, nodal_diag);
+         HYPRE_BoomerAMGSetCycleNumSweeps(pcg_precond, ns_coarse, 3);
          if (num_functions > 1)
             HYPRE_BoomerAMGSetDofFunc(pcg_precond, dof_func);
+         HYPRE_BoomerAMGSetAdditive(pcg_precond, additive);
+         HYPRE_BoomerAMGSetMultAdditive(pcg_precond, mult_add);
+         HYPRE_BoomerAMGSetSimple(pcg_precond, simple);
+         HYPRE_BoomerAMGSetMultAddPMaxElmts(pcg_precond, add_P_max_elmts);
+         HYPRE_BoomerAMGSetMultAddTruncFactor(pcg_precond, add_trunc_factor);
+         HYPRE_BoomerAMGSetRAP2(pcg_precond, rap2);
+         HYPRE_BoomerAMGSetKeepTranspose(pcg_precond, keepTranspose);
+         if (nongalerk_tol)
+         {
+            HYPRE_BoomerAMGSetNonGalerkinTol(pcg_precond, nongalerk_tol[nongalerk_num_tol-1]);
+            for (i=0; i < nongalerk_num_tol-1; i++)
+               HYPRE_BoomerAMGSetLevelNonGalerkinTol(pcg_precond, nongalerk_tol[i], i);
+         }
+         HYPRE_PCGSetMaxIter(pcg_solver, mg_max_iter);
          HYPRE_PCGSetPrecond(pcg_solver,
                              (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
                              (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
-                                   pcg_precond);
+                             pcg_precond);
       }
       else if (solver_id == 43)
       {
@@ -2281,7 +3036,15 @@ main( hypre_int argc,
             we'll use what I think is simplest: let Euclid internally 
             parse the command line.
          */   
-         HYPRE_EuclidSetParams(pcg_precond, argc, argv);
+         if (eu_level > -1) HYPRE_EuclidSetLevel(pcg_precond, eu_level);
+         if (eu_ilut) HYPRE_EuclidSetILUT(pcg_precond, eu_ilut);
+         if (eu_sparse_A) HYPRE_EuclidSetSparseA(pcg_precond, eu_sparse_A);
+         if (eu_row_scale) HYPRE_EuclidSetRowScale(pcg_precond, eu_row_scale);
+         if (eu_bj) HYPRE_EuclidSetBJ(pcg_precond, eu_bj);
+         HYPRE_EuclidSetStats(pcg_precond, eu_stats);
+         HYPRE_EuclidSetMem(pcg_precond, eu_mem);
+
+         /*HYPRE_EuclidSetParams(pcg_precond, argc, argv);*/
 
          HYPRE_PCGSetPrecond(pcg_solver,
                              (HYPRE_PtrToSolverFcn) HYPRE_EuclidSolve,
@@ -2292,15 +3055,15 @@ main( hypre_int argc,
       HYPRE_PCGGetPrecond(pcg_solver, &pcg_precond_gotten);
       if (pcg_precond_gotten !=  pcg_precond)
       {
-        hypre_printf("HYPRE_ParCSRPCGGetPrecond got bad precond\n");
-        return(-1);
+         hypre_printf("HYPRE_ParCSRPCGGetPrecond got bad precond\n");
+         return(-1);
       }
       else 
-        if (myid == 0)
-          hypre_printf("HYPRE_ParCSRPCGGetPrecond got good precond\n");
+         if (myid == 0)
+            hypre_printf("HYPRE_ParCSRPCGGetPrecond got good precond\n");
 
       HYPRE_PCGSetup(pcg_solver, (HYPRE_Matrix)parcsr_A, 
-		(HYPRE_Vector)b, (HYPRE_Vector)x);
+                     (HYPRE_Vector)b, (HYPRE_Vector)x);
  
       hypre_EndTiming(time_index);
       hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
@@ -2311,7 +3074,7 @@ main( hypre_int argc,
       hypre_BeginTiming(time_index);
  
       HYPRE_PCGSolve(pcg_solver, (HYPRE_Matrix)parcsr_A, 
-		(HYPRE_Vector)b, (HYPRE_Vector)x);
+                     (HYPRE_Vector)b, (HYPRE_Vector)x);
  
       hypre_EndTiming(time_index);
       hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
@@ -2325,9 +3088,9 @@ main( hypre_int argc,
       /* run a second time to check for memory leaks */
       HYPRE_ParVectorSetRandomValues(x, 775);
       HYPRE_PCGSetup(pcg_solver, (HYPRE_Matrix)parcsr_A, 
-		(HYPRE_Vector)b, (HYPRE_Vector)x);
+                     (HYPRE_Vector)b, (HYPRE_Vector)x);
       HYPRE_PCGSolve(pcg_solver, (HYPRE_Matrix)parcsr_A, 
-		(HYPRE_Vector)b, (HYPRE_Vector)x);
+                     (HYPRE_Vector)b, (HYPRE_Vector)x);
 #endif
 
       HYPRE_ParCSRPCGDestroy(pcg_solver);
@@ -2350,7 +3113,7 @@ main( hypre_int argc,
       }
       else if (solver_id == 43)
       {
-        HYPRE_EuclidDestroy(pcg_precond);
+         HYPRE_EuclidDestroy(pcg_precond);
       }
 
       if (myid == 0)
@@ -2366,8 +3129,8 @@ main( hypre_int argc,
    /* begin lobpcg */
 
    /*-----------------------------------------------------------
-    * Solve the eigenvalue problem using LOBPCG 
-    *-----------------------------------------------------------*/
+ *     * Solve the eigenvalue problem using LOBPCG 
+ *         *-----------------------------------------------------------*/
 
    if ( lobpcgFlag ) {
 
@@ -2383,96 +3146,96 @@ main( hypre_int argc,
        BuildParIsoLaplacian(argc, argv, &parcsr_B);
 
        ierr = HYPRE_ParCSRMatrixGetLocalRange( parcsr_B,
-					       &first_local_row, &last_local_row ,
-					       &first_local_col, &last_local_col );
+                                               &first_local_row, &last_local_row ,
+                                               &first_local_col, &last_local_col );
 
        local_num_rows = last_local_row - first_local_row + 1;
        local_num_cols = last_local_col - first_local_col + 1;
        ierr += HYPRE_ParCSRMatrixGetDims( parcsr_B, &M, &N );
 
        ierr += HYPRE_IJMatrixCreate( comm, first_local_row, last_local_row,
-				     first_local_col, last_local_col,
-				     &ij_B );
-      
-       ierr += HYPRE_IJMatrixSetObjectType( ij_B, HYPRE_PARCSR );  
-    
-       if (sparsity_known == 1) {   
-	 diag_sizes = hypre_CTAlloc(HYPRE_Int, local_num_rows);
-	 offdiag_sizes = hypre_CTAlloc(HYPRE_Int, local_num_rows);
-	 local_row = 0;
-	 for (i=first_local_row; i<= last_local_row; i++) {
-	   ierr += HYPRE_ParCSRMatrixGetRow( parcsr_B, i, &size, 
-					     &col_inds, &values );
-	   for (j=0; j < size; j++)
-	     {
-	       if (col_inds[j] < first_local_row || col_inds[j] > last_local_row)
-		 offdiag_sizes[local_row]++;
-	       else
-		 diag_sizes[local_row]++;
-	     }
-	   local_row++;
-	   ierr += HYPRE_ParCSRMatrixRestoreRow( parcsr_B, i, &size, 
-						 &col_inds, &values );
-	 }
-	 ierr += HYPRE_IJMatrixSetDiagOffdSizes( ij_B, 
-						 (const HYPRE_Int *) diag_sizes,
-						 (const HYPRE_Int *) offdiag_sizes );
-	 hypre_TFree(diag_sizes);
-	 hypre_TFree(offdiag_sizes);
-     
-	 ierr = HYPRE_IJMatrixInitialize( ij_B );
+                                     first_local_col, last_local_col,
+                                     &ij_B );
+
+       ierr += HYPRE_IJMatrixSetObjectType( ij_B, HYPRE_PARCSR );
+
+       if (sparsity_known == 1) {
+         diag_sizes = hypre_CTAlloc(HYPRE_Int, local_num_rows);
+         offdiag_sizes = hypre_CTAlloc(HYPRE_Int, local_num_rows);
+         local_row = 0;
+         for (i=first_local_row; i<= last_local_row; i++) {
+           ierr += HYPRE_ParCSRMatrixGetRow( parcsr_B, i, &size,
+                                             &col_inds, &values );
+           for (j=0; j < size; j++)
+             {
+               if (col_inds[j] < first_local_row || col_inds[j] > last_local_row)
+                 offdiag_sizes[local_row]++;
+               else
+                 diag_sizes[local_row]++;
+             }
+           local_row++;
+           ierr += HYPRE_ParCSRMatrixRestoreRow( parcsr_B, i, &size,
+                                                 &col_inds, &values );
+         }
+         ierr += HYPRE_IJMatrixSetDiagOffdSizes( ij_B,
+                                                 (const HYPRE_Int *) diag_sizes,
+                                                 (const HYPRE_Int *) offdiag_sizes );
+         hypre_TFree(diag_sizes);
+         hypre_TFree(offdiag_sizes);
 
-	 for (i=first_local_row; i<= last_local_row; i++)
-	   {
-	     ierr += HYPRE_ParCSRMatrixGetRow( parcsr_B, i, &size,
-					       &col_inds, &values );
+         ierr = HYPRE_IJMatrixInitialize( ij_B );
 
-	     ierr += HYPRE_IJMatrixSetValues( ij_B, 1, &size, &i,
-					      (const HYPRE_Int *) col_inds,
-					      (const HYPRE_Real *) values );
+         for (i=first_local_row; i<= last_local_row; i++)
+           {
+             ierr += HYPRE_ParCSRMatrixGetRow( parcsr_B, i, &size,
+                                               &col_inds, &values );
+
+             ierr += HYPRE_IJMatrixSetValues( ij_B, 1, &size, &i,
+                                              (const HYPRE_Int *) col_inds,
+                                              (const HYPRE_Real *) values );
 
-	     ierr += HYPRE_ParCSRMatrixRestoreRow( parcsr_B, i, &size,
-						   &col_inds, &values );
-	   }
+             ierr += HYPRE_ParCSRMatrixRestoreRow( parcsr_B, i, &size,
+                                                   &col_inds, &values );
+           }
        }
        else
-	 {
-	   row_sizes = hypre_CTAlloc(HYPRE_Int, local_num_rows);
-	
-	   size = 5; /* this is in general too low, and supposed to test
-			the capability of the reallocation of the interface */ 
+         {
+           row_sizes = hypre_CTAlloc(HYPRE_Int, local_num_rows);
 
-	   if (sparsity_known == 0) /* tries a more accurate estimate of the
-				       storage */
-	     {
-	       if (build_matrix_type == 2) size = 7;
-	       if (build_matrix_type == 3) size = 9;
-	       if (build_matrix_type == 4) size = 27;
-	     }
+           size = 5; /* this is in general too low, and supposed to test
+                        the capability of the reallocation of the interface */
 
-	   for (i=0; i < local_num_rows; i++)
-	     row_sizes[i] = size;
+           if (sparsity_known == 0) /* tries a more accurate estimate of the
+                                       storage */
+             {
+               if (build_matrix_type == 2) size = 7;
+               if (build_matrix_type == 3) size = 9;
+               if (build_matrix_type == 4) size = 27;
+             }
 
-	   ierr = HYPRE_IJMatrixSetRowSizes ( ij_B, (const HYPRE_Int *) row_sizes );
+           for (i=0; i < local_num_rows; i++)
+             row_sizes[i] = size;
 
-	   hypre_TFree(row_sizes);
+           ierr = HYPRE_IJMatrixSetRowSizes ( ij_B, (const HYPRE_Int *) row_sizes );
 
-	   ierr = HYPRE_IJMatrixInitialize( ij_B );
+           hypre_TFree(row_sizes);
 
-	   /* Loop through all locally stored rows and insert them into ij_matrix */
-	   for (i=first_local_row; i<= last_local_row; i++)
-	     {
-	       ierr += HYPRE_ParCSRMatrixGetRow( parcsr_B, i, &size,
-						 &col_inds, &values );
+           ierr = HYPRE_IJMatrixInitialize( ij_B );
 
-	       ierr += HYPRE_IJMatrixSetValues( ij_B, 1, &size, &i,
-						(const HYPRE_Int *) col_inds,
-						(const HYPRE_Real *) values );
+           /* Loop through all locally stored rows and insert them into ij_matrix */
+           for (i=first_local_row; i<= last_local_row; i++)
+             {
+               ierr += HYPRE_ParCSRMatrixGetRow( parcsr_B, i, &size,
+                                                 &col_inds, &values );
 
-	       ierr += HYPRE_ParCSRMatrixRestoreRow( parcsr_B, i, &size,
-						     &col_inds, &values );
-	     }
-	 }
+               ierr += HYPRE_IJMatrixSetValues( ij_B, 1, &size, &i,
+                                                (const HYPRE_Int *) col_inds,
+                                                (const HYPRE_Real *) values );
+
+               ierr += HYPRE_ParCSRMatrixRestoreRow( parcsr_B, i, &size,
+                                                     &col_inds, &values );
+             }
+         }
 
        ierr += HYPRE_IJMatrixAssemble( ij_B );
 
@@ -2483,6 +3246,7 @@ main( hypre_int argc,
 
      } /* if ( lobpcgGen ) */
 
+
      if ( pcgIterations > 0 ) { /* do inner pcg iterations */
 
        time_index = hypre_InitializeTiming("PCG Setup");
@@ -2494,275 +3258,260 @@ main( hypre_int argc,
        HYPRE_PCGSetTwoNorm(pcg_solver, two_norm);
        HYPRE_PCGSetRelChange(pcg_solver, 0);
        HYPRE_PCGSetPrintLevel(pcg_solver, 0);
- 
+
        HYPRE_PCGGetPrecond(pcg_solver, &pcg_precond);
 
        if (solver_id == 1)
-	 {
-	   /* use BoomerAMG as preconditioner */
-	   if (myid == 0) hypre_printf("Solver: AMG-PCG\n");
-	   HYPRE_BoomerAMGCreate(&pcg_precond); 
+         {
+           /* use BoomerAMG as preconditioner */
+           if (myid == 0) hypre_printf("Solver: AMG-PCG\n");
+           HYPRE_BoomerAMGCreate(&pcg_precond);
            /* BM Aug 25, 2006 */
            HYPRE_BoomerAMGSetCGCIts(pcg_precond, cgcits);
-	   HYPRE_BoomerAMGSetInterpType(pcg_precond, interp_type);
-	   HYPRE_BoomerAMGSetNumSamples(pcg_precond, gsmg_samples);
-	   HYPRE_BoomerAMGSetTol(pcg_precond, pc_tol);
-	   HYPRE_BoomerAMGSetCoarsenType(pcg_precond, (hybrid*coarsen_type));
-	   HYPRE_BoomerAMGSetMeasureType(pcg_precond, measure_type);
-	   HYPRE_BoomerAMGSetStrongThreshold(pcg_precond, strong_threshold);
-	   HYPRE_BoomerAMGSetTruncFactor(pcg_precond, trunc_factor);
+           HYPRE_BoomerAMGSetInterpType(pcg_precond, interp_type);
+           HYPRE_BoomerAMGSetNumSamples(pcg_precond, gsmg_samples);
+           HYPRE_BoomerAMGSetTol(pcg_precond, pc_tol);
+           HYPRE_BoomerAMGSetCoarsenType(pcg_precond, (hybrid*coarsen_type));
+           HYPRE_BoomerAMGSetMeasureType(pcg_precond, measure_type);
+           HYPRE_BoomerAMGSetStrongThreshold(pcg_precond, strong_threshold);
+           HYPRE_BoomerAMGSetTruncFactor(pcg_precond, trunc_factor);
            HYPRE_BoomerAMGSetPMaxElmts(pcg_precond, P_max_elmts);
            HYPRE_BoomerAMGSetPostInterpType(pcg_precond, post_interp_type);
            HYPRE_BoomerAMGSetJacobiTruncThreshold(pcg_precond, jacobi_trunc_threshold);
-	   HYPRE_BoomerAMGSetPrintLevel(pcg_precond, poutdat);
-	   HYPRE_BoomerAMGSetPrintFileName(pcg_precond, "driver.out.log");
-	   HYPRE_BoomerAMGSetMaxIter(pcg_precond, 1);
-	   HYPRE_BoomerAMGSetCycleType(pcg_precond, cycle_type);
-	   HYPRE_BoomerAMGSetNumGridSweeps(pcg_precond, num_grid_sweeps);
-	   HYPRE_BoomerAMGSetGridRelaxType(pcg_precond, grid_relax_type);
-	   HYPRE_BoomerAMGSetRelaxWeight(pcg_precond, relax_weight);
-	   HYPRE_BoomerAMGSetOmega(pcg_precond, omega);
-	   HYPRE_BoomerAMGSetSmoothType(pcg_precond, smooth_type);
-	   HYPRE_BoomerAMGSetSmoothNumLevels(pcg_precond, smooth_num_levels);
-	   HYPRE_BoomerAMGSetSmoothNumSweeps(pcg_precond, smooth_num_sweeps);
-	   HYPRE_BoomerAMGSetGridRelaxPoints(pcg_precond, grid_relax_points);
-	   HYPRE_BoomerAMGSetMaxLevels(pcg_precond, max_levels);
-	   HYPRE_BoomerAMGSetMaxRowSum(pcg_precond, max_row_sum);
-	   HYPRE_BoomerAMGSetNumFunctions(pcg_precond, num_functions);
+           HYPRE_BoomerAMGSetPrintLevel(pcg_precond, poutdat);
+           HYPRE_BoomerAMGSetPrintFileName(pcg_precond, "driver.out.log");
+           HYPRE_BoomerAMGSetMaxIter(pcg_precond, 1);
+           HYPRE_BoomerAMGSetCycleType(pcg_precond, cycle_type);
+           HYPRE_BoomerAMGSetNumSweeps(pcg_precond, num_sweeps);
+           if (relax_type > -1) HYPRE_BoomerAMGSetRelaxType(pcg_precond, relax_type);
+           if (relax_down > -1)
+              HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_down, 1);
+           if (relax_up > -1)
+              HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_up, 2);
+           if (relax_coarse > -1)
+              HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_coarse, 3);
+           HYPRE_BoomerAMGSetAddRelaxType(pcg_precond, add_relax_type);
+           HYPRE_BoomerAMGSetAddRelaxWt(pcg_precond, add_relax_wt);
+           HYPRE_BoomerAMGSetRelaxWt(pcg_precond, relax_wt);
+           HYPRE_BoomerAMGSetOuterWt(pcg_precond, outer_wt);
+           HYPRE_BoomerAMGSetSmoothType(pcg_precond, smooth_type);
+           HYPRE_BoomerAMGSetSmoothNumLevels(pcg_precond, smooth_num_levels);
+           HYPRE_BoomerAMGSetSmoothNumSweeps(pcg_precond, smooth_num_sweeps);
+           HYPRE_BoomerAMGSetRelaxOrder(pcg_precond, relax_order);
+           HYPRE_BoomerAMGSetMaxLevels(pcg_precond, max_levels);
+           HYPRE_BoomerAMGSetMaxRowSum(pcg_precond, max_row_sum);
+           HYPRE_BoomerAMGSetDebugFlag(pcg_precond, debug_flag);
+           HYPRE_BoomerAMGSetNumFunctions(pcg_precond, num_functions);
            HYPRE_BoomerAMGSetNumPaths(pcg_precond, num_paths);
            HYPRE_BoomerAMGSetAggNumLevels(pcg_precond, agg_num_levels);
-	   HYPRE_BoomerAMGSetVariant(pcg_precond, variant);
-	   HYPRE_BoomerAMGSetOverlap(pcg_precond, overlap);
-	   HYPRE_BoomerAMGSetDomainType(pcg_precond, domain_type);
-	   HYPRE_BoomerAMGSetSchwarzRlxWeight(pcg_precond, schwarz_rlx_weight);
-	   if (num_functions > 1)
-	     HYPRE_BoomerAMGSetDofFunc(pcg_precond, dof_func);
-	   HYPRE_PCGSetPrecond(pcg_solver,
-			       (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
-			       (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
-			       pcg_precond);
-	 }
+           HYPRE_BoomerAMGSetVariant(pcg_precond, variant);
+           HYPRE_BoomerAMGSetOverlap(pcg_precond, overlap);
+           HYPRE_BoomerAMGSetDomainType(pcg_precond, domain_type);
+           HYPRE_BoomerAMGSetSchwarzRlxWeight(pcg_precond, schwarz_rlx_weight);
+           if (num_functions > 1)
+             HYPRE_BoomerAMGSetDofFunc(pcg_precond, dof_func);
+           HYPRE_PCGSetPrecond(pcg_solver,
+                               (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
+                               (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
+                               pcg_precond);
+         }
        else if (solver_id == 2)
-	 {
-	    
-	   /* use diagonal scaling as preconditioner */
-	   if (myid == 0) hypre_printf("Solver: DS-PCG\n");
-	   pcg_precond = NULL;
-	    
-	   HYPRE_PCGSetPrecond(pcg_solver,
-			       (HYPRE_PtrToSolverFcn) HYPRE_ParCSRDiagScale,
-			       (HYPRE_PtrToSolverFcn) HYPRE_ParCSRDiagScaleSetup,
-			       pcg_precond);
-	 }
+         {
+
+           /* use diagonal scaling as preconditioner */
+           if (myid == 0) hypre_printf("Solver: DS-PCG\n");
+           pcg_precond = NULL;
+
+           HYPRE_PCGSetPrecond(pcg_solver,
+                               (HYPRE_PtrToSolverFcn) HYPRE_ParCSRDiagScale,
+                               (HYPRE_PtrToSolverFcn) HYPRE_ParCSRDiagScaleSetup,
+                               pcg_precond);
+         }
        else if (solver_id == 8)
-	 {
-	   /* use ParaSails preconditioner */
-	   if (myid == 0) hypre_printf("Solver: ParaSails-PCG\n");
-	    
-	   HYPRE_ParaSailsCreate(hypre_MPI_COMM_WORLD, &pcg_precond);
-	   HYPRE_ParaSailsSetParams(pcg_precond, sai_threshold, max_levels);
-	   HYPRE_ParaSailsSetFilter(pcg_precond, sai_filter);
-	   HYPRE_ParaSailsSetLogging(pcg_precond, poutdat);
-	    
-	   HYPRE_PCGSetPrecond(pcg_solver,
-			       (HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSolve,
-			       (HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSetup,
-			       pcg_precond);
-	 }
+         {
+           /* use ParaSails preconditioner */
+           if (myid == 0) hypre_printf("Solver: ParaSails-PCG\n");
+
+           HYPRE_ParaSailsCreate(hypre_MPI_COMM_WORLD, &pcg_precond);
+           HYPRE_ParaSailsSetParams(pcg_precond, sai_threshold, max_levels);
+           HYPRE_ParaSailsSetFilter(pcg_precond, sai_filter);
+           HYPRE_ParaSailsSetLogging(pcg_precond, poutdat);
+
+           HYPRE_PCGSetPrecond(pcg_solver,
+                               (HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSolve,
+                               (HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSetup,
+                               pcg_precond);
+         }
        else if (solver_id == 12)
-	 {
-	   /* use Schwarz preconditioner */
-	   if (myid == 0) hypre_printf("Solver: Schwarz-PCG\n");
-	    
-	   HYPRE_SchwarzCreate(&pcg_precond);
-	   HYPRE_SchwarzSetVariant(pcg_precond, variant);
-	   HYPRE_SchwarzSetOverlap(pcg_precond, overlap);
-	   HYPRE_SchwarzSetDomainType(pcg_precond, domain_type);
-	   HYPRE_SchwarzSetRelaxWeight(pcg_precond, schwarz_rlx_weight);
-	    
-	   HYPRE_PCGSetPrecond(pcg_solver,
-			       (HYPRE_PtrToSolverFcn) HYPRE_SchwarzSolve,
-			       (HYPRE_PtrToSolverFcn) HYPRE_SchwarzSetup,
-			       pcg_precond);
-	 }
+         {
+           /* use Schwarz preconditioner */
+           if (myid == 0) hypre_printf("Solver: Schwarz-PCG\n");
+
+           HYPRE_SchwarzCreate(&pcg_precond);
+           HYPRE_SchwarzSetVariant(pcg_precond, variant);
+           HYPRE_SchwarzSetOverlap(pcg_precond, overlap);
+           HYPRE_SchwarzSetDomainType(pcg_precond, domain_type);
+           HYPRE_SchwarzSetRelaxWeight(pcg_precond, schwarz_rlx_weight);
+
+           HYPRE_PCGSetPrecond(pcg_solver,
+                               (HYPRE_PtrToSolverFcn) HYPRE_SchwarzSolve,
+                               (HYPRE_PtrToSolverFcn) HYPRE_SchwarzSetup,
+                               pcg_precond);
+         }
        else if (solver_id == 14)
-	 {
-	   /* use GSMG as preconditioner */
-	    
-	   /* reset some smoother parameters */
-	    
-	   /* fine grid */
-	   num_grid_sweeps[0] = num_sweep;
-	   grid_relax_type[0] = relax_default;
-	   hypre_TFree (grid_relax_points[0]);
-	   grid_relax_points[0] = hypre_CTAlloc(HYPRE_Int, num_sweep);
-	   for (i=0; i<num_sweep; i++)
-	     grid_relax_points[0][i] = 0;
-	    
-	   /* down cycle */
-	   num_grid_sweeps[1] = num_sweep;
-	   grid_relax_type[1] = relax_default;
-	   hypre_TFree (grid_relax_points[1]);
-	   grid_relax_points[1] = hypre_CTAlloc(HYPRE_Int, num_sweep);
-	   for (i=0; i<num_sweep; i++)
-	     grid_relax_points[1][i] = 0;
-	    
-	   /* up cycle */
-	   num_grid_sweeps[2] = num_sweep;
-	   grid_relax_type[2] = relax_default;
-	   hypre_TFree (grid_relax_points[2]);
-	   grid_relax_points[2] = hypre_CTAlloc(HYPRE_Int, num_sweep);
-	   for (i=0; i<num_sweep; i++)
-	     grid_relax_points[2][i] = 0;
-	    
-	   /* coarsest grid */
-	   num_grid_sweeps[3] = 1;
-	   grid_relax_type[3] = 9;
-	   hypre_TFree (grid_relax_points[3]);
-	   grid_relax_points[3] = hypre_CTAlloc(HYPRE_Int, 1);
-	   grid_relax_points[3][0] = 0;
-	    
-	   if (myid == 0) hypre_printf("Solver: GSMG-PCG\n");
-	   HYPRE_BoomerAMGCreate(&pcg_precond); 
+         {
+           /* use GSMG as preconditioner */
+
+           /* reset some smoother parameters */
+
+           num_sweeps = num_sweep;
+           relax_type = relax_default;
+           relax_order = 0;
+
+           if (myid == 0) hypre_printf("Solver: GSMG-PCG\n");
+           HYPRE_BoomerAMGCreate(&pcg_precond);
            /* BM Aug 25, 2006 */
            HYPRE_BoomerAMGSetCGCIts(pcg_precond, cgcits);
-	   HYPRE_BoomerAMGSetGSMG(pcg_precond, 4); 
-	   HYPRE_BoomerAMGSetInterpType(pcg_precond, interp_type);
-	   HYPRE_BoomerAMGSetNumSamples(pcg_precond, gsmg_samples);
-	   HYPRE_BoomerAMGSetTol(pcg_precond, pc_tol);
-	   HYPRE_BoomerAMGSetCoarsenType(pcg_precond, (hybrid*coarsen_type));
-	   HYPRE_BoomerAMGSetMeasureType(pcg_precond, measure_type);
-	   HYPRE_BoomerAMGSetStrongThreshold(pcg_precond, strong_threshold);
-	   HYPRE_BoomerAMGSetTruncFactor(pcg_precond, trunc_factor);
+           HYPRE_BoomerAMGSetGSMG(pcg_precond, 4);
+           HYPRE_BoomerAMGSetInterpType(pcg_precond, interp_type);
+           HYPRE_BoomerAMGSetNumSamples(pcg_precond, gsmg_samples);
+           HYPRE_BoomerAMGSetTol(pcg_precond, pc_tol);
+           HYPRE_BoomerAMGSetCoarsenType(pcg_precond, (hybrid*coarsen_type));
+           HYPRE_BoomerAMGSetMeasureType(pcg_precond, measure_type);
+           HYPRE_BoomerAMGSetStrongThreshold(pcg_precond, strong_threshold);
+           HYPRE_BoomerAMGSetTruncFactor(pcg_precond, trunc_factor);
            HYPRE_BoomerAMGSetPMaxElmts(pcg_precond, P_max_elmts);
            HYPRE_BoomerAMGSetPostInterpType(pcg_precond, post_interp_type);
            HYPRE_BoomerAMGSetJacobiTruncThreshold(pcg_precond, jacobi_trunc_threshold);
-	   HYPRE_BoomerAMGSetPrintLevel(pcg_precond, poutdat);
-	   HYPRE_BoomerAMGSetPrintFileName(pcg_precond, "driver.out.log");
-	   HYPRE_BoomerAMGSetMaxIter(pcg_precond, 1);
-	   HYPRE_BoomerAMGSetCycleType(pcg_precond, cycle_type);
-	   HYPRE_BoomerAMGSetNumGridSweeps(pcg_precond, num_grid_sweeps);
-	   HYPRE_BoomerAMGSetGridRelaxType(pcg_precond, grid_relax_type);
-	   HYPRE_BoomerAMGSetRelaxWeight(pcg_precond, relax_weight);
-	   HYPRE_BoomerAMGSetOmega(pcg_precond, omega);
-	   HYPRE_BoomerAMGSetSmoothType(pcg_precond, smooth_type);
-	   HYPRE_BoomerAMGSetSmoothNumLevels(pcg_precond, smooth_num_levels);
-	   HYPRE_BoomerAMGSetSmoothNumSweeps(pcg_precond, smooth_num_sweeps);
-	   HYPRE_BoomerAMGSetVariant(pcg_precond, variant);
-	   HYPRE_BoomerAMGSetOverlap(pcg_precond, overlap);
-	   HYPRE_BoomerAMGSetDomainType(pcg_precond, domain_type);
-	   HYPRE_BoomerAMGSetSchwarzRlxWeight(pcg_precond, schwarz_rlx_weight);
-	   HYPRE_BoomerAMGSetGridRelaxPoints(pcg_precond, grid_relax_points);
-	   HYPRE_BoomerAMGSetMaxLevels(pcg_precond, max_levels);
-	   HYPRE_BoomerAMGSetMaxRowSum(pcg_precond, max_row_sum);
-	   HYPRE_BoomerAMGSetNumFunctions(pcg_precond, num_functions);
+           HYPRE_BoomerAMGSetPrintLevel(pcg_precond, poutdat);
+           HYPRE_BoomerAMGSetPrintFileName(pcg_precond, "driver.out.log");
+           HYPRE_BoomerAMGSetMaxIter(pcg_precond, 1);
+           HYPRE_BoomerAMGSetCycleType(pcg_precond, cycle_type);
+           HYPRE_BoomerAMGSetNumSweeps(pcg_precond, num_sweeps);
+           HYPRE_BoomerAMGSetRelaxType(pcg_precond, relax_type);
+           HYPRE_BoomerAMGSetRelaxWt(pcg_precond, relax_wt);
+           HYPRE_BoomerAMGSetAddRelaxType(pcg_precond, add_relax_type);
+           HYPRE_BoomerAMGSetAddRelaxWt(pcg_precond, add_relax_wt);
+           HYPRE_BoomerAMGSetOuterWt(pcg_precond, outer_wt);
+           HYPRE_BoomerAMGSetSmoothType(pcg_precond, smooth_type);
+           HYPRE_BoomerAMGSetSmoothNumLevels(pcg_precond, smooth_num_levels);
+           HYPRE_BoomerAMGSetSmoothNumSweeps(pcg_precond, smooth_num_sweeps);
+           HYPRE_BoomerAMGSetVariant(pcg_precond, variant);
+           HYPRE_BoomerAMGSetOverlap(pcg_precond, overlap);
+           HYPRE_BoomerAMGSetDomainType(pcg_precond, domain_type);
+           HYPRE_BoomerAMGSetSchwarzRlxWeight(pcg_precond, schwarz_rlx_weight);
+           HYPRE_BoomerAMGSetRelaxOrder(pcg_precond, relax_order);
+           HYPRE_BoomerAMGSetMaxLevels(pcg_precond, max_levels);
+           HYPRE_BoomerAMGSetMaxRowSum(pcg_precond, max_row_sum);
+           HYPRE_BoomerAMGSetDebugFlag(pcg_precond, debug_flag);
+           HYPRE_BoomerAMGSetNumFunctions(pcg_precond, num_functions);
            HYPRE_BoomerAMGSetNumPaths(pcg_precond, num_paths);
            HYPRE_BoomerAMGSetAggNumLevels(pcg_precond, agg_num_levels);
-	   if (num_functions > 1)
-	     HYPRE_BoomerAMGSetDofFunc(pcg_precond, dof_func);
-	   HYPRE_PCGSetPrecond(pcg_solver,
-			       (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
-			       (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
-			       pcg_precond);
-	 }
+           if (num_functions > 1)
+             HYPRE_BoomerAMGSetDofFunc(pcg_precond, dof_func);
+           HYPRE_PCGSetPrecond(pcg_solver,
+                               (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
+                               (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
+                               pcg_precond);
+         }
        else if (solver_id == 43)
-	 {
-	   /* use Euclid preconditioning */
-	   if (myid == 0) hypre_printf("Solver: Euclid-PCG\n");
-	    
-	   HYPRE_EuclidCreate(hypre_MPI_COMM_WORLD, &pcg_precond);
-	    
-	   /* note: There are three three methods of setting run-time 
-	      parameters for Euclid: (see HYPRE_parcsr_ls.h); here
-	      we'll use what I think is simplest: let Euclid internally 
-	      parse the command line.
-	   */   
-	   HYPRE_EuclidSetParams(pcg_precond, argc, argv);
-	    
-	   HYPRE_PCGSetPrecond(pcg_solver,
-			       (HYPRE_PtrToSolverFcn) HYPRE_EuclidSolve,
-			       (HYPRE_PtrToSolverFcn) HYPRE_EuclidSetup,
-			       pcg_precond);
-	 }
+         {
+           /* use Euclid preconditioning */
+           if (myid == 0) hypre_printf("Solver: Euclid-PCG\n");
+
+           HYPRE_EuclidCreate(hypre_MPI_COMM_WORLD, &pcg_precond);
+
+           /* note: There are three three methods of setting run-time 
+ *               parameters for Euclid: (see HYPRE_parcsr_ls.h); here
+ *                             we'll use what I think is simplest: let Euclid internally 
+ *                                           parse the command line.
+ *                                                      */
+           HYPRE_EuclidSetParams(pcg_precond, argc, argv);
+
+           HYPRE_PCGSetPrecond(pcg_solver,
+                               (HYPRE_PtrToSolverFcn) HYPRE_EuclidSolve,
+                               (HYPRE_PtrToSolverFcn) HYPRE_EuclidSetup,
+                               pcg_precond);
+         }
        else if (solver_id != NO_SOLVER )
-	 {
-	   if ( verbosity )
-	     hypre_printf("Solver ID not recognized - running inner PCG iterations without preconditioner\n\n");
-	 }
-	
+         {
+           if ( verbosity )
+             hypre_printf("Solver ID not recognized - running inner PCG iterations without preconditioner\n\n");
+         }
+
        HYPRE_PCGGetPrecond(pcg_solver, &pcg_precond_gotten);
        if (pcg_precond_gotten !=  pcg_precond)
-	 {
-	   hypre_printf("HYPRE_ParCSRPCGGetPrecond got bad precond\n");
-	   return(-1);
-	 }
-       else 
-	 if (myid == 0)
-	   hypre_printf("HYPRE_ParCSRPCGGetPrecond got good precond\n");
-	
+         {
+           hypre_printf("HYPRE_ParCSRPCGGetPrecond got bad precond\n");
+           return(-1);
+         }
+       else
+         if (myid == 0)
+           hypre_printf("HYPRE_ParCSRPCGGetPrecond got good precond\n");
+
        /*      HYPRE_PCGSetup(pcg_solver, (HYPRE_Matrix)parcsr_A, 
-	       (HYPRE_Vector)b, (HYPRE_Vector)x); */
-	
+ *                     (HYPRE_Vector)b, (HYPRE_Vector)x); */
+
        hypre_EndTiming(time_index);
        hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
        hypre_FinalizeTiming(time_index);
        hypre_ClearTiming();
-	
+
        HYPRE_LOBPCGCreate(interpreter, &matvec_fn, &lobpcg_solver);
 
        HYPRE_LOBPCGSetMaxIter(lobpcg_solver, maxIterations);
        HYPRE_LOBPCGSetPrecondUsageMode(lobpcg_solver, pcgMode);
        HYPRE_LOBPCGSetTol(lobpcg_solver, tol);
        HYPRE_LOBPCGSetPrintLevel(lobpcg_solver, verbosity);
-	
+
        HYPRE_LOBPCGSetPrecond(lobpcg_solver,
-			      (HYPRE_PtrToSolverFcn) HYPRE_PCGSolve,
-			      (HYPRE_PtrToSolverFcn) HYPRE_PCGSetup,
-			      pcg_solver);
+                              (HYPRE_PtrToSolverFcn) HYPRE_PCGSolve,
+                              (HYPRE_PtrToSolverFcn) HYPRE_PCGSetup,
+                              pcg_solver);
 
-       HYPRE_LOBPCGSetupT(lobpcg_solver, (HYPRE_Matrix)parcsr_A, 
-			  (HYPRE_Vector)x);
+       HYPRE_LOBPCGSetupT(lobpcg_solver, (HYPRE_Matrix)parcsr_A,
+                          (HYPRE_Vector)x);
 
-       HYPRE_LOBPCGSetup(lobpcg_solver, (HYPRE_Matrix)parcsr_A, 
-			 (HYPRE_Vector)b, (HYPRE_Vector)x);
+       HYPRE_LOBPCGSetup(lobpcg_solver, (HYPRE_Matrix)parcsr_A,
+                         (HYPRE_Vector)b, (HYPRE_Vector)x);
 
        if ( lobpcgGen )
-	 HYPRE_LOBPCGSetupB(lobpcg_solver, (HYPRE_Matrix)parcsr_B, 
-			    (HYPRE_Vector)x);
+         HYPRE_LOBPCGSetupB(lobpcg_solver, (HYPRE_Matrix)parcsr_B,
+                            (HYPRE_Vector)x);
 
        if ( vFromFileFlag ) {
-	 eigenvectors = mv_MultiVectorWrap( interpreter,
-                         hypre_ParCSRMultiVectorRead(hypre_MPI_COMM_WORLD, 
-					             interpreter, 
-					             "vectors" ),1);
-	 hypre_assert( eigenvectors != NULL );
-	 blockSize = mv_MultiVectorWidth( eigenvectors );
+         eigenvectors = mv_MultiVectorWrap( interpreter,
+                         hypre_ParCSRMultiVectorRead(hypre_MPI_COMM_WORLD,
+                                                     interpreter,
+                                                     "vectors" ),1);
+         hypre_assert( eigenvectors != NULL );
+         blockSize = mv_MultiVectorWidth( eigenvectors );
        }
        else {
-	 eigenvectors = mv_MultiVectorCreateFromSampleVector( interpreter,
-								 blockSize, 
-								 x );
-	 if ( lobpcgSeed )
-	   mv_MultiVectorSetRandom( eigenvectors, lobpcgSeed );
-	 else
-	   mv_MultiVectorSetRandom( eigenvectors, (HYPRE_Int)time(0) );
+         eigenvectors = mv_MultiVectorCreateFromSampleVector( interpreter,
+                                                                 blockSize,
+                                                                 x );
+         if ( lobpcgSeed )
+           mv_MultiVectorSetRandom( eigenvectors, lobpcgSeed );
+         else
+           mv_MultiVectorSetRandom( eigenvectors, (HYPRE_Int)time(0) );
        }
 
        if ( constrained ) {
-	 constraints = mv_MultiVectorWrap( interpreter,
+         constraints = mv_MultiVectorWrap( interpreter,
                          hypre_ParCSRMultiVectorRead(hypre_MPI_COMM_WORLD,
                                                      interpreter,
                                                      "vectors" ),1);
-	 hypre_assert( constraints != NULL );
-       }	
+         hypre_assert( constraints != NULL );
+       }
 
        eigenvalues = (HYPRE_Real*) calloc( blockSize, sizeof(HYPRE_Real) );
-	
+
        time_index = hypre_InitializeTiming("LOBPCG Solve");
        hypre_BeginTiming(time_index);
-	
+
        HYPRE_LOBPCGSolve(lobpcg_solver, constraints, eigenvectors, eigenvalues );
-	
+
        hypre_EndTiming(time_index);
        hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
        hypre_FinalizeTiming(time_index);
@@ -2770,340 +3519,322 @@ main( hypre_int argc,
 
 
        if ( checkOrtho ) {
-		
-	 gramXX = utilities_FortranMatrixCreate();
-	 identity = utilities_FortranMatrixCreate();
-
-	 utilities_FortranMatrixAllocateData( blockSize, blockSize, gramXX );
-	 utilities_FortranMatrixAllocateData( blockSize, blockSize, identity );
-
-	 if ( lobpcgGen ) {
-	   workspace = mv_MultiVectorCreateCopy( eigenvectors, 0 );
-	   hypre_LOBPCGMultiOperatorB( lobpcg_solver, 
-	                               mv_MultiVectorGetData(eigenvectors),
-	                               mv_MultiVectorGetData(workspace) );
-	   lobpcg_MultiVectorByMultiVector( eigenvectors, workspace, gramXX );
-	 }
-	 else
-	   lobpcg_MultiVectorByMultiVector( eigenvectors, eigenvectors, gramXX );
-
-	 utilities_FortranMatrixSetToIdentity( identity );
-	 utilities_FortranMatrixAdd( -1, identity, gramXX, gramXX );
-	 nonOrthF = utilities_FortranMatrixFNorm( gramXX );
-	 if ( myid == 0 )
-	   hypre_printf("Non-orthonormality of eigenvectors: %12.5e\n", nonOrthF);
-		
-	 utilities_FortranMatrixDestroy( gramXX );
-	 utilities_FortranMatrixDestroy( identity );
-	  
+
+         gramXX = utilities_FortranMatrixCreate();
+         identity = utilities_FortranMatrixCreate();
+
+         utilities_FortranMatrixAllocateData( blockSize, blockSize, gramXX );
+         utilities_FortranMatrixAllocateData( blockSize, blockSize, identity );
+
+         if ( lobpcgGen ) {
+           workspace = mv_MultiVectorCreateCopy( eigenvectors, 0 );
+           hypre_LOBPCGMultiOperatorB( lobpcg_solver,
+                                       mv_MultiVectorGetData(eigenvectors),
+                                       mv_MultiVectorGetData(workspace) );
+           lobpcg_MultiVectorByMultiVector( eigenvectors, workspace, gramXX );
+         }
+         else
+           lobpcg_MultiVectorByMultiVector( eigenvectors, eigenvectors, gramXX );
+
+         utilities_FortranMatrixSetToIdentity( identity );
+         utilities_FortranMatrixAdd( -1, identity, gramXX, gramXX );
+         nonOrthF = utilities_FortranMatrixFNorm( gramXX );
+         if ( myid == 0 )
+           hypre_printf("Non-orthonormality of eigenvectors: %12.5e\n", nonOrthF);
+
+         utilities_FortranMatrixDestroy( gramXX );
+         utilities_FortranMatrixDestroy( identity );
+
        }
 
        if ( printLevel ) {
-		  
-	 hypre_ParCSRMultiVectorPrint( mv_MultiVectorGetData(eigenvectors), "vectors" );
-
-	 if ( myid == 0 ) {	  
-	   if ( (filePtr = fopen("values.txt", "w")) ) {
-	     hypre_fprintf(filePtr, "%d\n", blockSize);
-	     for ( i = 0; i < blockSize; i++ )
-	       hypre_fprintf(filePtr, "%22.14e\n", eigenvalues[i]);
-	     fclose(filePtr);
-	   }
-	    
-	   if ( (filePtr = fopen("residuals.txt", "w")) ) {
-	     residualNorms = HYPRE_LOBPCGResidualNorms( lobpcg_solver );
-	     residuals = utilities_FortranMatrixValues( residualNorms );
-	     hypre_fprintf(filePtr, "%d\n", blockSize);
-	     for ( i = 0; i < blockSize; i++ )
-	       hypre_fprintf(filePtr, "%22.14e\n", residuals[i]);
-	     fclose(filePtr);
-	   }
-	    
-	   if ( printLevel > 1 ) {
-	      
-	     printBuffer = utilities_FortranMatrixCreate();
-	    
-	     iterations = HYPRE_LOBPCGIterations( lobpcg_solver );
-
-	     eigenvaluesHistory = HYPRE_LOBPCGEigenvaluesHistory( lobpcg_solver );
-	     utilities_FortranMatrixSelectBlock( eigenvaluesHistory,
-						 1, blockSize, 1, iterations + 1, printBuffer );
-	     utilities_FortranMatrixPrint( printBuffer, "val_hist.txt" );
-
-	     residualNormsHistory = HYPRE_LOBPCGResidualNormsHistory( lobpcg_solver );
-	     utilities_FortranMatrixSelectBlock(residualNormsHistory, 
-						1, blockSize, 1, iterations + 1, printBuffer );
-	     utilities_FortranMatrixPrint( printBuffer, "res_hist.txt" );
-	      
-	     utilities_FortranMatrixDestroy( printBuffer );
-	   }
-	 }
+
+         hypre_ParCSRMultiVectorPrint( mv_MultiVectorGetData(eigenvectors), "vectors" );
+
+         if ( myid == 0 ) {
+           if ( (filePtr = fopen("values.txt", "w")) ) {
+             hypre_fprintf(filePtr, "%d\n", blockSize);
+             for ( i = 0; i < blockSize; i++ )
+               hypre_fprintf(filePtr, "%22.14e\n", eigenvalues[i]);
+             fclose(filePtr);
+           }
+
+           if ( (filePtr = fopen("residuals.txt", "w")) ) {
+             residualNorms = HYPRE_LOBPCGResidualNorms( lobpcg_solver );
+             residuals = utilities_FortranMatrixValues( residualNorms );
+             hypre_fprintf(filePtr, "%d\n", blockSize);
+             for ( i = 0; i < blockSize; i++ )
+               hypre_fprintf(filePtr, "%22.14e\n", residuals[i]);
+             fclose(filePtr);
+           }
+
+           if ( printLevel > 1 ) {
+
+             printBuffer = utilities_FortranMatrixCreate();
+
+             iterations = HYPRE_LOBPCGIterations( lobpcg_solver );
+
+             eigenvaluesHistory = HYPRE_LOBPCGEigenvaluesHistory( lobpcg_solver );
+             utilities_FortranMatrixSelectBlock( eigenvaluesHistory,
+                                                 1, blockSize, 1, iterations + 1, printBuffer );
+             utilities_FortranMatrixPrint( printBuffer, "val_hist.txt" );
+             residualNormsHistory = HYPRE_LOBPCGResidualNormsHistory( lobpcg_solver );
+             utilities_FortranMatrixSelectBlock(residualNormsHistory,
+                                                1, blockSize, 1, iterations + 1, printBuffer );
+             utilities_FortranMatrixPrint( printBuffer, "res_hist.txt" );
+
+             utilities_FortranMatrixDestroy( printBuffer );
+           }
+         }
        }
 
        HYPRE_LOBPCGDestroy(lobpcg_solver);
        mv_MultiVectorDestroy( eigenvectors );
        if ( constrained )
-	 mv_MultiVectorDestroy( constraints );
+         mv_MultiVectorDestroy( constraints );
        if ( lobpcgGen )
-	 mv_MultiVectorDestroy( workspace );
+         mv_MultiVectorDestroy( workspace );
        free( eigenvalues );
-	
+
        HYPRE_ParCSRPCGDestroy(pcg_solver);
-	
+
        if (solver_id == 1)
-	 {
-	   HYPRE_BoomerAMGDestroy(pcg_precond);
-	 }
+         {
+           HYPRE_BoomerAMGDestroy(pcg_precond);
+         }
        else if (solver_id == 8)
-	 {
-	   HYPRE_ParaSailsDestroy(pcg_precond);
-	 }
+         {
+           HYPRE_ParaSailsDestroy(pcg_precond);
+         }
        else if (solver_id == 12)
-	 {
-	   HYPRE_SchwarzDestroy(pcg_precond);
-	 }
+         {
+           HYPRE_SchwarzDestroy(pcg_precond);
+         }
        else if (solver_id == 14)
-	 {
-	   HYPRE_BoomerAMGDestroy(pcg_precond);
-	 }
+         {
+           HYPRE_BoomerAMGDestroy(pcg_precond);
+         }
        else if (solver_id == 43)
-	 {
-	   HYPRE_EuclidDestroy(pcg_precond);
-	 }
-	
+         {
+           HYPRE_EuclidDestroy(pcg_precond);
+         }
+
      }
      else { /* pcgIterations <= 0 --> use the preconditioner directly */
-	
+
        time_index = hypre_InitializeTiming("LOBPCG Setup");
        hypre_BeginTiming(time_index);
-       if (myid != 0) 
-	 verbosity = 0;
- 
+       if (myid != 0)
+         verbosity = 0;
        HYPRE_LOBPCGCreate(interpreter, &matvec_fn, &pcg_solver);
        HYPRE_LOBPCGSetMaxIter(pcg_solver, maxIterations);
        HYPRE_LOBPCGSetTol(pcg_solver, tol);
        HYPRE_LOBPCGSetPrintLevel(pcg_solver, verbosity);
-	
+
        HYPRE_LOBPCGGetPrecond(pcg_solver, &pcg_precond);
-	
+
        if (solver_id == 1)
-	 {
-	   /* use BoomerAMG as preconditioner */
-	   if (myid == 0) 
-	     hypre_printf("Solver: AMG-PCG\n");
-	    
-	   HYPRE_BoomerAMGCreate(&pcg_precond); 
+         {
+           /* use BoomerAMG as preconditioner */
+           if (myid == 0)
+             hypre_printf("Solver: AMG-PCG\n");
+
+           HYPRE_BoomerAMGCreate(&pcg_precond);
            /* BM Aug 25, 2006 */
            HYPRE_BoomerAMGSetCGCIts(pcg_precond, cgcits);
-	   HYPRE_BoomerAMGSetInterpType(pcg_precond, interp_type);
-	   HYPRE_BoomerAMGSetNumSamples(pcg_precond, gsmg_samples);
-	   HYPRE_BoomerAMGSetTol(pcg_precond, pc_tol);
-	   HYPRE_BoomerAMGSetCoarsenType(pcg_precond, (hybrid*coarsen_type));
-	   HYPRE_BoomerAMGSetMeasureType(pcg_precond, measure_type);
-	   HYPRE_BoomerAMGSetStrongThreshold(pcg_precond, strong_threshold);
-	   HYPRE_BoomerAMGSetTruncFactor(pcg_precond, trunc_factor);
+           HYPRE_BoomerAMGSetInterpType(pcg_precond, interp_type);
+           HYPRE_BoomerAMGSetNumSamples(pcg_precond, gsmg_samples);
+           HYPRE_BoomerAMGSetTol(pcg_precond, pc_tol);
+           HYPRE_BoomerAMGSetCoarsenType(pcg_precond, (hybrid*coarsen_type));
+           HYPRE_BoomerAMGSetMeasureType(pcg_precond, measure_type);
+           HYPRE_BoomerAMGSetStrongThreshold(pcg_precond, strong_threshold);
+           HYPRE_BoomerAMGSetTruncFactor(pcg_precond, trunc_factor);
            HYPRE_BoomerAMGSetPMaxElmts(pcg_precond, P_max_elmts);
            HYPRE_BoomerAMGSetPostInterpType(pcg_precond, post_interp_type);
            HYPRE_BoomerAMGSetJacobiTruncThreshold(pcg_precond, jacobi_trunc_threshold);
-	   HYPRE_BoomerAMGSetPrintLevel(pcg_precond, poutdat);
-	   HYPRE_BoomerAMGSetPrintFileName(pcg_precond, "driver.out.log");
-	   HYPRE_BoomerAMGSetMaxIter(pcg_precond, 1);
-	   HYPRE_BoomerAMGSetCycleType(pcg_precond, cycle_type);
-	   HYPRE_BoomerAMGSetNumGridSweeps(pcg_precond, num_grid_sweeps);
-	   HYPRE_BoomerAMGSetGridRelaxType(pcg_precond, grid_relax_type);
-	   HYPRE_BoomerAMGSetRelaxWeight(pcg_precond, relax_weight);
-	   HYPRE_BoomerAMGSetOmega(pcg_precond, omega);
-	   HYPRE_BoomerAMGSetSmoothType(pcg_precond, smooth_type);
-	   HYPRE_BoomerAMGSetSmoothNumLevels(pcg_precond, smooth_num_levels);
-	   HYPRE_BoomerAMGSetSmoothNumSweeps(pcg_precond, smooth_num_sweeps);
-	   HYPRE_BoomerAMGSetGridRelaxPoints(pcg_precond, grid_relax_points);
-	   HYPRE_BoomerAMGSetMaxLevels(pcg_precond, max_levels);
-	   HYPRE_BoomerAMGSetMaxRowSum(pcg_precond, max_row_sum);
-	   HYPRE_BoomerAMGSetNumFunctions(pcg_precond, num_functions);
+           HYPRE_BoomerAMGSetPrintLevel(pcg_precond, poutdat);
+           HYPRE_BoomerAMGSetPrintFileName(pcg_precond, "driver.out.log");
+           HYPRE_BoomerAMGSetMaxIter(pcg_precond, 1);
+           HYPRE_BoomerAMGSetCycleType(pcg_precond, cycle_type);
+           HYPRE_BoomerAMGSetNumSweeps(pcg_precond, num_sweeps);
+           if (relax_type > -1) HYPRE_BoomerAMGSetRelaxType(pcg_precond, relax_type);
+           if (relax_down > -1)
+              HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_down, 1);
+           if (relax_up > -1)
+              HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_up, 2);
+           if (relax_coarse > -1)
+              HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_coarse, 3);
+           HYPRE_BoomerAMGSetAddRelaxType(pcg_precond, add_relax_type);
+           HYPRE_BoomerAMGSetAddRelaxWt(pcg_precond, add_relax_wt);
+           HYPRE_BoomerAMGSetRelaxWt(pcg_precond, relax_wt);
+           HYPRE_BoomerAMGSetOuterWt(pcg_precond, outer_wt);
+           HYPRE_BoomerAMGSetSmoothType(pcg_precond, smooth_type);
+           HYPRE_BoomerAMGSetSmoothNumLevels(pcg_precond, smooth_num_levels);
+           HYPRE_BoomerAMGSetSmoothNumSweeps(pcg_precond, smooth_num_sweeps);
+           HYPRE_BoomerAMGSetRelaxOrder(pcg_precond, relax_order);
+           HYPRE_BoomerAMGSetMaxLevels(pcg_precond, max_levels);
+           HYPRE_BoomerAMGSetMaxRowSum(pcg_precond, max_row_sum);
+           HYPRE_BoomerAMGSetDebugFlag(pcg_precond, debug_flag);
+           HYPRE_BoomerAMGSetNumFunctions(pcg_precond, num_functions);
            HYPRE_BoomerAMGSetNumPaths(pcg_precond, num_paths);
            HYPRE_BoomerAMGSetAggNumLevels(pcg_precond, agg_num_levels);
-	   HYPRE_BoomerAMGSetVariant(pcg_precond, variant);
-	   HYPRE_BoomerAMGSetOverlap(pcg_precond, overlap);
-	   HYPRE_BoomerAMGSetDomainType(pcg_precond, domain_type);
-	   HYPRE_BoomerAMGSetSchwarzRlxWeight(pcg_precond, schwarz_rlx_weight);
-	   if (num_functions > 1)
-	     HYPRE_BoomerAMGSetDofFunc(pcg_precond, dof_func);
-	    
-	   HYPRE_LOBPCGSetPrecond(pcg_solver,
-				  (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
-				  (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
-				  pcg_precond);
-	 }
+           HYPRE_BoomerAMGSetVariant(pcg_precond, variant);
+           HYPRE_BoomerAMGSetOverlap(pcg_precond, overlap);
+           HYPRE_BoomerAMGSetDomainType(pcg_precond, domain_type);
+           HYPRE_BoomerAMGSetSchwarzRlxWeight(pcg_precond, schwarz_rlx_weight);
+           if (num_functions > 1)
+             HYPRE_BoomerAMGSetDofFunc(pcg_precond, dof_func);
+           HYPRE_LOBPCGSetPrecond(pcg_solver,
+                                  (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
+                                  (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
+                                  pcg_precond);
+         }
        else if (solver_id == 2)
-	 {
-	    
-	   /* use diagonal scaling as preconditioner */
-	   if (myid == 0) 
-	     hypre_printf("Solver: DS-PCG\n");
-         
-	   pcg_precond = NULL;
-	    
-	   HYPRE_LOBPCGSetPrecond(pcg_solver,
-				  (HYPRE_PtrToSolverFcn) HYPRE_ParCSRDiagScale,
-				  (HYPRE_PtrToSolverFcn) HYPRE_ParCSRDiagScaleSetup,
-				  pcg_precond);
-	 }
+         {
+
+           /* use diagonal scaling as preconditioner */
+           if (myid == 0)
+             hypre_printf("Solver: DS-PCG\n");
+
+           pcg_precond = NULL;
+
+           HYPRE_LOBPCGSetPrecond(pcg_solver,
+                                  (HYPRE_PtrToSolverFcn) HYPRE_ParCSRDiagScale,
+                                  (HYPRE_PtrToSolverFcn) HYPRE_ParCSRDiagScaleSetup,
+                                  pcg_precond);
+         }
        else if (solver_id == 8)
-	 {
-	   /* use ParaSails preconditioner */
-	   if (myid == 0) 
-	     hypre_printf("Solver: ParaSails-PCG\n");
-	 
-	   HYPRE_ParaSailsCreate(hypre_MPI_COMM_WORLD, &pcg_precond);
-	   HYPRE_ParaSailsSetParams(pcg_precond, sai_threshold, max_levels);
-	   HYPRE_ParaSailsSetFilter(pcg_precond, sai_filter);
-	   HYPRE_ParaSailsSetLogging(pcg_precond, poutdat);
-	    
-	   HYPRE_LOBPCGSetPrecond(pcg_solver,
-				  (HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSolve,
-				  (HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSetup,
-				  pcg_precond);
-	 }
+         {
+           /* use ParaSails preconditioner */
+           if (myid == 0)
+             hypre_printf("Solver: ParaSails-PCG\n");
+
+           HYPRE_ParaSailsCreate(hypre_MPI_COMM_WORLD, &pcg_precond);
+           HYPRE_ParaSailsSetParams(pcg_precond, sai_threshold, max_levels);
+           HYPRE_ParaSailsSetFilter(pcg_precond, sai_filter);
+           HYPRE_ParaSailsSetLogging(pcg_precond, poutdat);
+
+           HYPRE_LOBPCGSetPrecond(pcg_solver,
+                                  (HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSolve,
+                                  (HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSetup,
+                                  pcg_precond);
+         }
        else if (solver_id == 12)
-	 {
-	   /* use Schwarz preconditioner */
-	   if (myid == 0) 
-	     hypre_printf("Solver: Schwarz-PCG\n");
-	    
-	   HYPRE_SchwarzCreate(&pcg_precond);		
-	   HYPRE_SchwarzSetVariant(pcg_precond, variant);		
-	   HYPRE_SchwarzSetOverlap(pcg_precond, overlap);		
-	   HYPRE_SchwarzSetDomainType(pcg_precond, domain_type);
-	   HYPRE_SchwarzSetRelaxWeight(pcg_precond, schwarz_rlx_weight);
-	    
-	   HYPRE_LOBPCGSetPrecond(pcg_solver,
-				  (HYPRE_PtrToSolverFcn) HYPRE_SchwarzSolve,
-				  (HYPRE_PtrToSolverFcn) HYPRE_SchwarzSetup,
-				  pcg_precond);
-	 }
+         {
+           /* use Schwarz preconditioner */
+           if (myid == 0)
+             hypre_printf("Solver: Schwarz-PCG\n");
+
+           HYPRE_SchwarzCreate(&pcg_precond);
+           HYPRE_SchwarzSetVariant(pcg_precond, variant);
+           HYPRE_SchwarzSetOverlap(pcg_precond, overlap);
+           HYPRE_SchwarzSetDomainType(pcg_precond, domain_type);
+           HYPRE_SchwarzSetRelaxWeight(pcg_precond, schwarz_rlx_weight);
+
+           HYPRE_LOBPCGSetPrecond(pcg_solver,
+                                  (HYPRE_PtrToSolverFcn) HYPRE_SchwarzSolve,
+                                  (HYPRE_PtrToSolverFcn) HYPRE_SchwarzSetup,
+                                  pcg_precond);
+         }
        else if (solver_id == 14)
-	 {
-	   /* use GSMG as preconditioner */
-	    
-	   /* reset some smoother parameters */
-	    
-	   /* fine grid */
-	   num_grid_sweeps[0] = num_sweep;
-	   grid_relax_type[0] = relax_default;
-	   hypre_TFree (grid_relax_points[0]);
-	   grid_relax_points[0] = hypre_CTAlloc(HYPRE_Int, num_sweep);
-	   for (i=0; i<num_sweep; i++)
-	     grid_relax_points[0][i] = 0;
-	    
-	   /* down cycle */
-	   num_grid_sweeps[1] = num_sweep;
-	   grid_relax_type[1] = relax_default;
-	   hypre_TFree (grid_relax_points[1]);
-	   grid_relax_points[1] = hypre_CTAlloc(HYPRE_Int, num_sweep);
-	   for (i=0; i<num_sweep; i++)
-	     grid_relax_points[1][i] = 0;
-	    
-	   /* up cycle */
-	   num_grid_sweeps[2] = num_sweep;
-	   grid_relax_type[2] = relax_default;
-	   hypre_TFree (grid_relax_points[2]);
-	   grid_relax_points[2] = hypre_CTAlloc(HYPRE_Int, num_sweep);
-	   for (i=0; i<num_sweep; i++)
-	     grid_relax_points[2][i] = 0;
-	    
-	   /* coarsest grid */
-	   num_grid_sweeps[3] = 1;
-	   grid_relax_type[3] = 9;
-	   hypre_TFree (grid_relax_points[3]);
-	   grid_relax_points[3] = hypre_CTAlloc(HYPRE_Int, 1);
-	   grid_relax_points[3][0] = 0;
-	    
-	   if (myid == 0) hypre_printf("Solver: GSMG-PCG\n");
-	   HYPRE_BoomerAMGCreate(&pcg_precond); 
+         {
+           /* use GSMG as preconditioner */
+
+           /* reset some smoother parameters */
+
+           num_sweeps = num_sweep;
+           relax_type = relax_default;
+           relax_order = 0;
+
+           if (myid == 0) hypre_printf("Solver: GSMG-PCG\n");
+           HYPRE_BoomerAMGCreate(&pcg_precond);
            /* BM Aug 25, 2006 */
            HYPRE_BoomerAMGSetCGCIts(pcg_precond, cgcits);
-	   HYPRE_BoomerAMGSetGSMG(pcg_precond, 4); 
-	   HYPRE_BoomerAMGSetInterpType(pcg_precond, interp_type);
-	   HYPRE_BoomerAMGSetNumSamples(pcg_precond, gsmg_samples);
-	   HYPRE_BoomerAMGSetTol(pcg_precond, pc_tol);
-	   HYPRE_BoomerAMGSetCoarsenType(pcg_precond, (hybrid*coarsen_type));
-	   HYPRE_BoomerAMGSetMeasureType(pcg_precond, measure_type);
-	   HYPRE_BoomerAMGSetStrongThreshold(pcg_precond, strong_threshold);
-	   HYPRE_BoomerAMGSetTruncFactor(pcg_precond, trunc_factor);
+           HYPRE_BoomerAMGSetGSMG(pcg_precond, 4);
+           HYPRE_BoomerAMGSetInterpType(pcg_precond, interp_type);
+           HYPRE_BoomerAMGSetNumSamples(pcg_precond, gsmg_samples);
+           HYPRE_BoomerAMGSetTol(pcg_precond, pc_tol);
+           HYPRE_BoomerAMGSetCoarsenType(pcg_precond, (hybrid*coarsen_type));
+           HYPRE_BoomerAMGSetMeasureType(pcg_precond, measure_type);
+           HYPRE_BoomerAMGSetStrongThreshold(pcg_precond, strong_threshold);
+           HYPRE_BoomerAMGSetTruncFactor(pcg_precond, trunc_factor);
            HYPRE_BoomerAMGSetPMaxElmts(pcg_precond, P_max_elmts);
            HYPRE_BoomerAMGSetPostInterpType(pcg_precond, post_interp_type);
            HYPRE_BoomerAMGSetJacobiTruncThreshold(pcg_precond, jacobi_trunc_threshold);
-	   HYPRE_BoomerAMGSetPrintLevel(pcg_precond, poutdat);
-	   HYPRE_BoomerAMGSetPrintFileName(pcg_precond, "driver.out.log");
-	   HYPRE_BoomerAMGSetMaxIter(pcg_precond, 1);
-	   HYPRE_BoomerAMGSetCycleType(pcg_precond, cycle_type);
-	   HYPRE_BoomerAMGSetNumGridSweeps(pcg_precond, num_grid_sweeps);
-	   HYPRE_BoomerAMGSetGridRelaxType(pcg_precond, grid_relax_type);
-	   HYPRE_BoomerAMGSetRelaxWeight(pcg_precond, relax_weight);
-	   HYPRE_BoomerAMGSetOmega(pcg_precond, omega);
-	   HYPRE_BoomerAMGSetSmoothType(pcg_precond, smooth_type);
-	   HYPRE_BoomerAMGSetSmoothNumLevels(pcg_precond, smooth_num_levels);
-	   HYPRE_BoomerAMGSetSmoothNumSweeps(pcg_precond, smooth_num_sweeps);
-	   HYPRE_BoomerAMGSetVariant(pcg_precond, variant);
-	   HYPRE_BoomerAMGSetOverlap(pcg_precond, overlap);
-	   HYPRE_BoomerAMGSetDomainType(pcg_precond, domain_type);
-	   HYPRE_BoomerAMGSetSchwarzRlxWeight(pcg_precond, schwarz_rlx_weight);
-	   HYPRE_BoomerAMGSetGridRelaxPoints(pcg_precond, grid_relax_points);
-	   HYPRE_BoomerAMGSetMaxLevels(pcg_precond, max_levels);
-	   HYPRE_BoomerAMGSetMaxRowSum(pcg_precond, max_row_sum);
-	   HYPRE_BoomerAMGSetNumFunctions(pcg_precond, num_functions);
+           HYPRE_BoomerAMGSetPrintLevel(pcg_precond, poutdat);
+           HYPRE_BoomerAMGSetPrintFileName(pcg_precond, "driver.out.log");
+           HYPRE_BoomerAMGSetMaxIter(pcg_precond, 1);
+           HYPRE_BoomerAMGSetCycleType(pcg_precond, cycle_type);
+           HYPRE_BoomerAMGSetNumSweeps(pcg_precond, num_sweeps);
+           HYPRE_BoomerAMGSetRelaxType(pcg_precond, relax_type);
+           HYPRE_BoomerAMGSetAddRelaxType(pcg_precond, add_relax_type);
+           HYPRE_BoomerAMGSetAddRelaxWt(pcg_precond, add_relax_wt);
+           HYPRE_BoomerAMGSetRelaxWt(pcg_precond, relax_wt);
+           HYPRE_BoomerAMGSetOuterWt(pcg_precond, outer_wt);
+           HYPRE_BoomerAMGSetSmoothType(pcg_precond, smooth_type);
+           HYPRE_BoomerAMGSetSmoothNumLevels(pcg_precond, smooth_num_levels);
+           HYPRE_BoomerAMGSetSmoothNumSweeps(pcg_precond, smooth_num_sweeps);
+           HYPRE_BoomerAMGSetVariant(pcg_precond, variant);
+           HYPRE_BoomerAMGSetOverlap(pcg_precond, overlap);
+           HYPRE_BoomerAMGSetDomainType(pcg_precond, domain_type);
+           HYPRE_BoomerAMGSetSchwarzRlxWeight(pcg_precond, schwarz_rlx_weight);
+           HYPRE_BoomerAMGSetRelaxOrder(pcg_precond, relax_order);
+           HYPRE_BoomerAMGSetMaxLevels(pcg_precond, max_levels);
+           HYPRE_BoomerAMGSetMaxRowSum(pcg_precond, max_row_sum);
+           HYPRE_BoomerAMGSetDebugFlag(pcg_precond, debug_flag);
+           HYPRE_BoomerAMGSetNumFunctions(pcg_precond, num_functions);
            HYPRE_BoomerAMGSetNumPaths(pcg_precond, num_paths);
            HYPRE_BoomerAMGSetAggNumLevels(pcg_precond, agg_num_levels);
-	   if (num_functions > 1)
-	     HYPRE_BoomerAMGSetDofFunc(pcg_precond, dof_func);
-	    
-	   HYPRE_LOBPCGSetPrecond(pcg_solver,
-				  (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
-				  (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
-				  pcg_precond);
-	 }
+           if (num_functions > 1)
+             HYPRE_BoomerAMGSetDofFunc(pcg_precond, dof_func);
+
+           HYPRE_LOBPCGSetPrecond(pcg_solver,
+                                  (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
+                                  (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
+                                  pcg_precond);
+         }
        else if (solver_id == 43)
-	 {
-	   /* use Euclid preconditioning */
-	   if (myid == 0) 
-	     hypre_printf("Solver: Euclid-PCG\n");
-	    
-	   HYPRE_EuclidCreate(hypre_MPI_COMM_WORLD, &pcg_precond);
-	    
-	   /* note: There are three three methods of setting run-time 
-	      parameters for Euclid: (see HYPRE_parcsr_ls.h); here
-	      we'll use what I think is simplest: let Euclid internally 
-	      parse the command line.
-	   */   
-	   HYPRE_EuclidSetParams(pcg_precond, argc, argv);
-	    
-	   HYPRE_LOBPCGSetPrecond(pcg_solver,
-				  (HYPRE_PtrToSolverFcn) HYPRE_EuclidSolve,
-				  (HYPRE_PtrToSolverFcn) HYPRE_EuclidSetup,
-				  pcg_precond);
-	 }
+         {
+           /* use Euclid preconditioning */
+           if (myid == 0)
+             hypre_printf("Solver: Euclid-PCG\n");
+
+           HYPRE_EuclidCreate(hypre_MPI_COMM_WORLD, &pcg_precond);
+
+           /* note: There are three three methods of setting run-time 
+ *               parameters for Euclid: (see HYPRE_parcsr_ls.h); here
+ *                             we'll use what I think is simplest: let Euclid internally 
+ *                                           parse the command line.
+ *                                                      */
+           HYPRE_EuclidSetParams(pcg_precond, argc, argv);
+
+           HYPRE_LOBPCGSetPrecond(pcg_solver,
+                                  (HYPRE_PtrToSolverFcn) HYPRE_EuclidSolve,
+                                  (HYPRE_PtrToSolverFcn) HYPRE_EuclidSetup,
+                                  pcg_precond);
+         }
        else if (solver_id != NO_SOLVER )
-	 {
-	   if ( verbosity )
-	     hypre_printf("Solver ID not recognized - running LOBPCG without preconditioner\n\n");
-	 }
-	
+         {
+           if ( verbosity )
+             hypre_printf("Solver ID not recognized - running LOBPCG without preconditioner\n\n");
+         }
+
        HYPRE_LOBPCGGetPrecond(pcg_solver, &pcg_precond_gotten);
        if (pcg_precond_gotten !=  pcg_precond && pcgIterations)
-	 {
-	   hypre_printf("HYPRE_ParCSRLOBPCGGetPrecond got bad precond\n");
-	   return(-1);
-	 }
-       else 
-	 if (myid == 0)
-	   hypre_printf("HYPRE_ParCSRLOBPCGGetPrecond got good precond\n");
-	
-       HYPRE_LOBPCGSetup(pcg_solver, (HYPRE_Matrix)parcsr_A, 
-			 (HYPRE_Vector)b, (HYPRE_Vector)x);
- 
+         {
+           hypre_printf("HYPRE_ParCSRLOBPCGGetPrecond got bad precond\n");
+           return(-1);
+         }
+       else
+         if (myid == 0)
+           hypre_printf("HYPRE_ParCSRLOBPCGGetPrecond got good precond\n");
+
+       HYPRE_LOBPCGSetup(pcg_solver, (HYPRE_Matrix)parcsr_A,
+                         (HYPRE_Vector)b, (HYPRE_Vector)x);
+
        if ( lobpcgGen )
-	 HYPRE_LOBPCGSetupB(pcg_solver, (HYPRE_Matrix)parcsr_B, 
-			    (HYPRE_Vector)x);
+         HYPRE_LOBPCGSetupB(pcg_solver, (HYPRE_Matrix)parcsr_B,
+                            (HYPRE_Vector)x);
 
        hypre_EndTiming(time_index);
        hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
@@ -3115,148 +3846,147 @@ main( hypre_int argc,
                          hypre_ParCSRMultiVectorRead(hypre_MPI_COMM_WORLD,
                                                      interpreter,
                                                      "vectors" ),1);
-	 hypre_assert( eigenvectors != NULL );
-	 blockSize = mv_MultiVectorWidth( eigenvectors );
+         hypre_assert( eigenvectors != NULL );
+         blockSize = mv_MultiVectorWidth( eigenvectors );
        }
        else {
-	 eigenvectors = mv_MultiVectorCreateFromSampleVector( interpreter,
-								 blockSize, 
-								 x );
-	 if ( lobpcgSeed )
-	   mv_MultiVectorSetRandom( eigenvectors, lobpcgSeed );
-	 else
-	   mv_MultiVectorSetRandom( eigenvectors, (HYPRE_Int)time(0) );
+         eigenvectors = mv_MultiVectorCreateFromSampleVector( interpreter,
+                                                                 blockSize,
+                                                                 x );
+         if ( lobpcgSeed )
+           mv_MultiVectorSetRandom( eigenvectors, lobpcgSeed );
+         else
+           mv_MultiVectorSetRandom( eigenvectors, (HYPRE_Int)time(0) );
        }
-	
+
        if ( constrained ) {
          constraints = mv_MultiVectorWrap( interpreter,
                          hypre_ParCSRMultiVectorRead(hypre_MPI_COMM_WORLD,
                                                      interpreter,
                                                      "vectors" ),1);
-	 hypre_assert( constraints != NULL );
-       }	
+         hypre_assert( constraints != NULL );
+       }
 
        eigenvalues = (HYPRE_Real*) calloc( blockSize, sizeof(HYPRE_Real) );
-	
+
        time_index = hypre_InitializeTiming("LOBPCG Solve");
        hypre_BeginTiming(time_index);
-	
+
        HYPRE_LOBPCGSolve(pcg_solver, constraints, eigenvectors, eigenvalues);
-	
+
        hypre_EndTiming(time_index);
        hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
        hypre_FinalizeTiming(time_index);
-       hypre_ClearTiming(); 
-	
+       hypre_ClearTiming();
+
        if ( checkOrtho ) {
-	  
-	 gramXX = utilities_FortranMatrixCreate();
-	 identity = utilities_FortranMatrixCreate();
 
-	 utilities_FortranMatrixAllocateData( blockSize, blockSize, gramXX );
-	 utilities_FortranMatrixAllocateData( blockSize, blockSize, identity );
+         gramXX = utilities_FortranMatrixCreate();
+         identity = utilities_FortranMatrixCreate();
+
+         utilities_FortranMatrixAllocateData( blockSize, blockSize, gramXX );
+         utilities_FortranMatrixAllocateData( blockSize, blockSize, identity );
 
-	 if ( lobpcgGen ) {
-	   workspace = mv_MultiVectorCreateCopy( eigenvectors, 0 );
+         if ( lobpcgGen ) {
+           workspace = mv_MultiVectorCreateCopy( eigenvectors, 0 );
            hypre_LOBPCGMultiOperatorB( pcg_solver,
                                        mv_MultiVectorGetData(eigenvectors),
                                        mv_MultiVectorGetData(workspace) );
-	   lobpcg_MultiVectorByMultiVector( eigenvectors, workspace, gramXX );
-	 }
-	 else
-	   lobpcg_MultiVectorByMultiVector( eigenvectors, eigenvectors, gramXX );
-
-	 utilities_FortranMatrixSetToIdentity( identity );
-	 utilities_FortranMatrixAdd( -1, identity, gramXX, gramXX );
-	 nonOrthF = utilities_FortranMatrixFNorm( gramXX );
-	 if ( myid == 0 )
-	   hypre_printf("Non-orthonormality of eigenvectors: %12.5e\n", nonOrthF);
-		
-	 utilities_FortranMatrixDestroy( gramXX );
-	 utilities_FortranMatrixDestroy( identity );
-	  
+           lobpcg_MultiVectorByMultiVector( eigenvectors, workspace, gramXX );
+         }
+         else
+           lobpcg_MultiVectorByMultiVector( eigenvectors, eigenvectors, gramXX );
+
+         utilities_FortranMatrixSetToIdentity( identity );
+         utilities_FortranMatrixAdd( -1, identity, gramXX, gramXX );
+         nonOrthF = utilities_FortranMatrixFNorm( gramXX );
+         if ( myid == 0 )
+           hypre_printf("Non-orthonormality of eigenvectors: %12.5e\n", nonOrthF);
+
+         utilities_FortranMatrixDestroy( gramXX );
+         utilities_FortranMatrixDestroy( identity );
+
        }
 
        if ( printLevel ) {
-	  
          hypre_ParCSRMultiVectorPrint( mv_MultiVectorGetData(eigenvectors), "vectors" );
-	  
-	 if ( myid == 0 ) {
-	   if ( (filePtr = fopen("values.txt", "w")) ) {
-	     hypre_fprintf(filePtr, "%d\n", blockSize);
-	     for ( i = 0; i < blockSize; i++ )
-	       hypre_fprintf(filePtr, "%22.14e\n", eigenvalues[i]);
-	     fclose(filePtr);
-	   }
-	    
-	   if ( (filePtr = fopen("residuals.txt", "w")) ) {
-	     residualNorms = HYPRE_LOBPCGResidualNorms( pcg_solver );
-	     residuals = utilities_FortranMatrixValues( residualNorms );
-	     hypre_fprintf(filePtr, "%d\n", blockSize);
-	     for ( i = 0; i < blockSize; i++ )
-	       hypre_fprintf(filePtr, "%22.14e\n", residuals[i]);
-	     fclose(filePtr);
-	   }
-
-	   if ( printLevel > 1 ) {
-
-	     printBuffer = utilities_FortranMatrixCreate();
-
-	     iterations = HYPRE_LOBPCGIterations( pcg_solver );
-
-	     eigenvaluesHistory = HYPRE_LOBPCGEigenvaluesHistory( pcg_solver );
-	     utilities_FortranMatrixSelectBlock( eigenvaluesHistory,
-						 1, blockSize, 1, iterations + 1, printBuffer );
-	     utilities_FortranMatrixPrint( printBuffer, "val_hist.txt" );
-
-	     residualNormsHistory = HYPRE_LOBPCGResidualNormsHistory( pcg_solver );
-	     utilities_FortranMatrixSelectBlock(residualNormsHistory,
-						1, blockSize, 1, iterations + 1, printBuffer );
-	     utilities_FortranMatrixPrint( printBuffer, "res_hist.txt" );
-	      
-	     utilities_FortranMatrixDestroy( printBuffer );
-	   }
-	 }
-       } 
+
+         if ( myid == 0 ) {
+           if ( (filePtr = fopen("values.txt", "w")) ) {
+             hypre_fprintf(filePtr, "%d\n", blockSize);
+             for ( i = 0; i < blockSize; i++ )
+               hypre_fprintf(filePtr, "%22.14e\n", eigenvalues[i]);
+             fclose(filePtr);
+           }
+
+           if ( (filePtr = fopen("residuals.txt", "w")) ) {
+             residualNorms = HYPRE_LOBPCGResidualNorms( pcg_solver );
+             residuals = utilities_FortranMatrixValues( residualNorms );
+             hypre_fprintf(filePtr, "%d\n", blockSize);
+             for ( i = 0; i < blockSize; i++ )
+               hypre_fprintf(filePtr, "%22.14e\n", residuals[i]);
+             fclose(filePtr);
+           }
+
+           if ( printLevel > 1 ) {
+
+             printBuffer = utilities_FortranMatrixCreate();
+
+             iterations = HYPRE_LOBPCGIterations( pcg_solver );
+
+             eigenvaluesHistory = HYPRE_LOBPCGEigenvaluesHistory( pcg_solver );
+             utilities_FortranMatrixSelectBlock( eigenvaluesHistory,
+                                                 1, blockSize, 1, iterations + 1, printBuffer );
+             utilities_FortranMatrixPrint( printBuffer, "val_hist.txt" );
+
+             residualNormsHistory = HYPRE_LOBPCGResidualNormsHistory( pcg_solver );
+             utilities_FortranMatrixSelectBlock(residualNormsHistory,
+                                                1, blockSize, 1, iterations + 1, printBuffer );
+             utilities_FortranMatrixPrint( printBuffer, "res_hist.txt" );
+
+             utilities_FortranMatrixDestroy( printBuffer );
+           }
+         }
+       }
 
 #if SECOND_TIME
        /* run a second time to check for memory leaks */
        mv_MultiVectorSetRandom( eigenvectors, 775 );
-       HYPRE_LOBPCGSetup(pcg_solver, (HYPRE_Matrix)parcsr_A, 
-			 (HYPRE_Vector)b, (HYPRE_Vector)x);
+       HYPRE_LOBPCGSetup(pcg_solver, (HYPRE_Matrix)parcsr_A,
+                         (HYPRE_Vector)b, (HYPRE_Vector)x);
        HYPRE_LOBPCGSolve(pcg_solver, constraints, eigenvectors, eigenvalues );
 #endif
 
        HYPRE_LOBPCGDestroy(pcg_solver);
- 
+
        if (solver_id == 1)
-	 {
-	   HYPRE_BoomerAMGDestroy(pcg_precond);
-	 }
+         {
+           HYPRE_BoomerAMGDestroy(pcg_precond);
+         }
        else if (solver_id == 8)
-	 {	 
-	   HYPRE_ParaSailsDestroy(pcg_precond);
-	 }
+         {
+           HYPRE_ParaSailsDestroy(pcg_precond);
+         }
        else if (solver_id == 12)
-	 {	 
-	   HYPRE_SchwarzDestroy(pcg_precond);
-	 }
+         {
+           HYPRE_SchwarzDestroy(pcg_precond);
+         }
        else if (solver_id == 14)
-	 {	 
-	   HYPRE_BoomerAMGDestroy(pcg_precond);
-	 }
+         {
+           HYPRE_BoomerAMGDestroy(pcg_precond);
+         }
        else if (solver_id == 43)
-	 {
-	   HYPRE_EuclidDestroy(pcg_precond);
-	 }
+         {
+           HYPRE_EuclidDestroy(pcg_precond);
+         }
 
        mv_MultiVectorDestroy( eigenvectors );
        if ( constrained )
-	 mv_MultiVectorDestroy( constraints );
+         mv_MultiVectorDestroy( constraints );
        if ( lobpcgGen )
-	 mv_MultiVectorDestroy( workspace );
+         mv_MultiVectorDestroy( workspace );
        free( eigenvalues );
-	
+
      } /* if ( pcgIterations > 0 ) */
 
      hypre_TFree( interpreter );
@@ -3280,51 +4010,112 @@ main( hypre_int argc,
  
       HYPRE_ParCSRGMRESCreate(hypre_MPI_COMM_WORLD, &pcg_solver);
       HYPRE_GMRESSetKDim(pcg_solver, k_dim);
-      HYPRE_GMRESSetMaxIter(pcg_solver, 1000);
+      HYPRE_GMRESSetMaxIter(pcg_solver, max_iter);
       HYPRE_GMRESSetTol(pcg_solver, tol);
+      HYPRE_GMRESSetAbsoluteTol(pcg_solver, atol);
       HYPRE_GMRESSetLogging(pcg_solver, 1);
       HYPRE_GMRESSetPrintLevel(pcg_solver, ioutdat);
- 
+      HYPRE_GMRESSetRelChange(pcg_solver, rel_change);
+
       if (solver_id == 3)
       {
          /* use BoomerAMG as preconditioner */
          if (myid == 0) hypre_printf("Solver: AMG-GMRES\n");
 
          HYPRE_BoomerAMGCreate(&pcg_precond); 
-         /* BM Aug 25, 2006 */
          HYPRE_BoomerAMGSetCGCIts(pcg_precond, cgcits);
          HYPRE_BoomerAMGSetInterpType(pcg_precond, interp_type);
+         HYPRE_BoomerAMGSetPostInterpType(pcg_precond, post_interp_type);
          HYPRE_BoomerAMGSetNumSamples(pcg_precond, gsmg_samples);
          HYPRE_BoomerAMGSetTol(pcg_precond, pc_tol);
-         HYPRE_BoomerAMGSetCoarsenType(pcg_precond, (hybrid*coarsen_type));
+         HYPRE_BoomerAMGSetCoarsenType(pcg_precond, coarsen_type);
          HYPRE_BoomerAMGSetMeasureType(pcg_precond, measure_type);
          HYPRE_BoomerAMGSetStrongThreshold(pcg_precond, strong_threshold);
+         HYPRE_BoomerAMGSetSeqThreshold(pcg_precond, seq_threshold);
+         HYPRE_BoomerAMGSetRedundant(pcg_precond, redundant);
+         HYPRE_BoomerAMGSetMaxCoarseSize(pcg_precond, coarse_threshold);
+         HYPRE_BoomerAMGSetMinCoarseSize(pcg_precond, min_coarse_size);
          HYPRE_BoomerAMGSetTruncFactor(pcg_precond, trunc_factor);
          HYPRE_BoomerAMGSetPMaxElmts(pcg_precond, P_max_elmts);
-         HYPRE_BoomerAMGSetPostInterpType(pcg_precond, post_interp_type);
          HYPRE_BoomerAMGSetJacobiTruncThreshold(pcg_precond, jacobi_trunc_threshold);
+         HYPRE_BoomerAMGSetSCommPkgSwitch(pcg_precond, S_commpkg_switch);
          HYPRE_BoomerAMGSetPrintLevel(pcg_precond, poutdat);
          HYPRE_BoomerAMGSetPrintFileName(pcg_precond, "driver.out.log");
          HYPRE_BoomerAMGSetMaxIter(pcg_precond, 1);
          HYPRE_BoomerAMGSetCycleType(pcg_precond, cycle_type);
-         HYPRE_BoomerAMGSetNumGridSweeps(pcg_precond, num_grid_sweeps);
-         HYPRE_BoomerAMGSetGridRelaxType(pcg_precond, grid_relax_type);
-         HYPRE_BoomerAMGSetRelaxWeight(pcg_precond, relax_weight);
-         HYPRE_BoomerAMGSetOmega(pcg_precond, omega);
+         HYPRE_BoomerAMGSetNumSweeps(pcg_precond, num_sweeps);
+         HYPRE_BoomerAMGSetISType(pcg_precond, IS_type);
+         HYPRE_BoomerAMGSetNumCRRelaxSteps(pcg_precond, num_CR_relax_steps);
+         HYPRE_BoomerAMGSetCRRate(pcg_precond, CR_rate);
+         HYPRE_BoomerAMGSetCRStrongTh(pcg_precond, CR_strong_th);
+         HYPRE_BoomerAMGSetCRUseCG(pcg_precond, CR_use_CG);
+         if (relax_type > -1) HYPRE_BoomerAMGSetRelaxType(pcg_precond, relax_type);
+         if (relax_down > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_down, 1);
+         if (relax_up > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_up, 2);
+         if (relax_coarse > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_coarse, 3);
+         HYPRE_BoomerAMGSetAddRelaxType(pcg_precond, add_relax_type);
+         HYPRE_BoomerAMGSetAddRelaxWt(pcg_precond, add_relax_wt);
+         HYPRE_BoomerAMGSetChebyOrder(pcg_precond, cheby_order);
+         HYPRE_BoomerAMGSetChebyFraction(pcg_precond, cheby_fraction);
+         HYPRE_BoomerAMGSetRelaxOrder(pcg_precond, relax_order);
+         HYPRE_BoomerAMGSetRelaxWt(pcg_precond, relax_wt);
+         HYPRE_BoomerAMGSetOuterWt(pcg_precond, outer_wt);
+         if (level_w > -1)
+            HYPRE_BoomerAMGSetLevelRelaxWt(pcg_precond, relax_wt_level,level_w);
+         if (level_ow > -1)
+            HYPRE_BoomerAMGSetLevelOuterWt(pcg_precond,outer_wt_level,level_ow);
          HYPRE_BoomerAMGSetSmoothType(pcg_precond, smooth_type);
          HYPRE_BoomerAMGSetSmoothNumLevels(pcg_precond, smooth_num_levels);
          HYPRE_BoomerAMGSetSmoothNumSweeps(pcg_precond, smooth_num_sweeps);
-         HYPRE_BoomerAMGSetGridRelaxPoints(pcg_precond, grid_relax_points);
          HYPRE_BoomerAMGSetMaxLevels(pcg_precond, max_levels);
          HYPRE_BoomerAMGSetMaxRowSum(pcg_precond, max_row_sum);
+         HYPRE_BoomerAMGSetDebugFlag(pcg_precond, debug_flag);
          HYPRE_BoomerAMGSetNumFunctions(pcg_precond, num_functions);
-         HYPRE_BoomerAMGSetNumPaths(pcg_precond, num_paths);
          HYPRE_BoomerAMGSetAggNumLevels(pcg_precond, agg_num_levels);
+         HYPRE_BoomerAMGSetAggInterpType(pcg_precond, agg_interp_type);
+         HYPRE_BoomerAMGSetAggTruncFactor(pcg_precond, agg_trunc_factor);
+         HYPRE_BoomerAMGSetAggP12TruncFactor(pcg_precond, agg_P12_trunc_factor);
+         HYPRE_BoomerAMGSetAggPMaxElmts(pcg_precond, agg_P_max_elmts);
+         HYPRE_BoomerAMGSetAggP12MaxElmts(pcg_precond, agg_P12_max_elmts);
+         HYPRE_BoomerAMGSetNumPaths(pcg_precond, num_paths);
+         HYPRE_BoomerAMGSetNodal(pcg_precond, nodal);
+         HYPRE_BoomerAMGSetNodalDiag(pcg_precond, nodal_diag);
          HYPRE_BoomerAMGSetVariant(pcg_precond, variant);
          HYPRE_BoomerAMGSetOverlap(pcg_precond, overlap);
          HYPRE_BoomerAMGSetDomainType(pcg_precond, domain_type);
+         HYPRE_BoomerAMGSetSchwarzUseNonSymm(pcg_precond, use_nonsymm_schwarz);
+         HYPRE_BoomerAMGSetSchwarzRlxWeight(pcg_precond, schwarz_rlx_weight);
+         if (eu_level < 0) eu_level = 0;
+         HYPRE_BoomerAMGSetEuLevel(pcg_precond, eu_level);
+         HYPRE_BoomerAMGSetEuBJ(pcg_precond, eu_bj);
+         HYPRE_BoomerAMGSetEuSparseA(pcg_precond, eu_sparse_A);
+         HYPRE_BoomerAMGSetCycleNumSweeps(pcg_precond, ns_coarse, 3);
          if (num_functions > 1)
             HYPRE_BoomerAMGSetDofFunc(pcg_precond, dof_func);
+         HYPRE_BoomerAMGSetAdditive(pcg_precond, additive);
+         HYPRE_BoomerAMGSetMultAdditive(pcg_precond, mult_add);
+         HYPRE_BoomerAMGSetSimple(pcg_precond, simple);
+         HYPRE_BoomerAMGSetMultAddPMaxElmts(pcg_precond, add_P_max_elmts);
+         HYPRE_BoomerAMGSetMultAddTruncFactor(pcg_precond, add_trunc_factor);
+         HYPRE_BoomerAMGSetRAP2(pcg_precond, rap2);
+         HYPRE_BoomerAMGSetKeepTranspose(pcg_precond, keepTranspose);
+         if (nongalerk_tol)
+         {
+            HYPRE_BoomerAMGSetNonGalerkinTol(pcg_precond, nongalerk_tol[nongalerk_num_tol-1]);
+            for (i=0; i < nongalerk_num_tol-1; i++)
+               HYPRE_BoomerAMGSetLevelNonGalerkinTol(pcg_precond, nongalerk_tol[i], i);
+         }
+         if (build_rbm)
+         {
+            HYPRE_BoomerAMGSetInterpVectors(pcg_precond, 1, interp_vecs);
+            HYPRE_BoomerAMGSetInterpVecVariant(pcg_precond, interp_vec_variant);
+            HYPRE_BoomerAMGSetInterpVecQMax(pcg_precond, Q_max);
+            HYPRE_BoomerAMGSetInterpVecAbsQTrunc(pcg_precond, Q_trunc);
+         }
+         HYPRE_GMRESSetMaxIter(pcg_solver, mg_max_iter);
          HYPRE_GMRESSetPrecond(pcg_solver,
                                (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
                                (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
@@ -3348,7 +4139,7 @@ main( hypre_int argc,
 
          ierr = HYPRE_ParCSRPilutCreate( hypre_MPI_COMM_WORLD, &pcg_precond ); 
          if (ierr) {
-	   hypre_printf("Error in ParPilutCreate\n");
+            hypre_printf("Error in ParPilutCreate\n");
          }
 
          HYPRE_GMRESSetPrecond(pcg_solver,
@@ -3358,11 +4149,11 @@ main( hypre_int argc,
 
          if (drop_tol >= 0 )
             HYPRE_ParCSRPilutSetDropTolerance( pcg_precond,
-               drop_tol );
+                                               drop_tol );
 
          if (nonzeros_to_keep >= 0 )
             HYPRE_ParCSRPilutSetFactorRowSize( pcg_precond,
-               nonzeros_to_keep );
+                                               nonzeros_to_keep );
       }
       else if (solver_id == 15)
       {
@@ -3370,79 +4161,101 @@ main( hypre_int argc,
 
          /* reset some smoother parameters */
     
-         /* fine grid */
-         num_grid_sweeps[0] = num_sweep;
-         grid_relax_type[0] = relax_default;
-         hypre_TFree (grid_relax_points[0]);
-         grid_relax_points[0] = hypre_CTAlloc(HYPRE_Int, num_sweep);
-         for (i=0; i<num_sweep; i++)
-            grid_relax_points[0][i] = 0;
-    
-         /* down cycle */
-         num_grid_sweeps[1] = num_sweep;
-         grid_relax_type[1] = relax_default;
-         hypre_TFree (grid_relax_points[1]);
-         grid_relax_points[1] = hypre_CTAlloc(HYPRE_Int, num_sweep);
-         for (i=0; i<num_sweep; i++)
-            grid_relax_points[1][i] = 0;
-    
-         /* up cycle */
-         num_grid_sweeps[2] = num_sweep;
-         grid_relax_type[2] = relax_default;
-         hypre_TFree (grid_relax_points[2]);
-         grid_relax_points[2] = hypre_CTAlloc(HYPRE_Int, num_sweep);
-         for (i=0; i<num_sweep; i++)
-            grid_relax_points[2][i] = 0;
-    
-         /* coarsest grid */
-         num_grid_sweeps[3] = 1;
-         grid_relax_type[3] = 9;
-         hypre_TFree (grid_relax_points[3]);
-         grid_relax_points[3] = hypre_CTAlloc(HYPRE_Int, 1);
-         grid_relax_points[3][0] = 0;
+         relax_order = 0;
  
          if (myid == 0) hypre_printf("Solver: GSMG-GMRES\n");
          HYPRE_BoomerAMGCreate(&pcg_precond); 
-         /* BM Aug 25, 2006 */
-         HYPRE_BoomerAMGSetCGCIts(pcg_precond, cgcits);
          HYPRE_BoomerAMGSetGSMG(pcg_precond, 4); 
+         HYPRE_BoomerAMGSetCGCIts(pcg_precond, cgcits);
          HYPRE_BoomerAMGSetInterpType(pcg_precond, interp_type);
+         HYPRE_BoomerAMGSetPostInterpType(pcg_precond, post_interp_type);
          HYPRE_BoomerAMGSetNumSamples(pcg_precond, gsmg_samples);
          HYPRE_BoomerAMGSetTol(pcg_precond, pc_tol);
-         HYPRE_BoomerAMGSetCoarsenType(pcg_precond, (hybrid*coarsen_type));
+         HYPRE_BoomerAMGSetCoarsenType(pcg_precond, coarsen_type);
          HYPRE_BoomerAMGSetMeasureType(pcg_precond, measure_type);
          HYPRE_BoomerAMGSetStrongThreshold(pcg_precond, strong_threshold);
+         HYPRE_BoomerAMGSetSeqThreshold(pcg_precond, seq_threshold);
+         HYPRE_BoomerAMGSetRedundant(pcg_precond, redundant);
+         HYPRE_BoomerAMGSetMaxCoarseSize(pcg_precond, coarse_threshold);
+         HYPRE_BoomerAMGSetMinCoarseSize(pcg_precond, min_coarse_size);
          HYPRE_BoomerAMGSetTruncFactor(pcg_precond, trunc_factor);
          HYPRE_BoomerAMGSetPMaxElmts(pcg_precond, P_max_elmts);
-         HYPRE_BoomerAMGSetPostInterpType(pcg_precond, post_interp_type);
          HYPRE_BoomerAMGSetJacobiTruncThreshold(pcg_precond, jacobi_trunc_threshold);
+         HYPRE_BoomerAMGSetSCommPkgSwitch(pcg_precond, S_commpkg_switch);
          HYPRE_BoomerAMGSetPrintLevel(pcg_precond, poutdat);
          HYPRE_BoomerAMGSetPrintFileName(pcg_precond, "driver.out.log");
          HYPRE_BoomerAMGSetMaxIter(pcg_precond, 1);
          HYPRE_BoomerAMGSetCycleType(pcg_precond, cycle_type);
-         HYPRE_BoomerAMGSetNumGridSweeps(pcg_precond, num_grid_sweeps);
-         HYPRE_BoomerAMGSetGridRelaxType(pcg_precond, grid_relax_type);
-         HYPRE_BoomerAMGSetRelaxWeight(pcg_precond, relax_weight);
-         HYPRE_BoomerAMGSetOmega(pcg_precond, omega);
+         HYPRE_BoomerAMGSetNumSweeps(pcg_precond, num_sweeps);
+         HYPRE_BoomerAMGSetISType(pcg_precond, IS_type);
+         HYPRE_BoomerAMGSetNumCRRelaxSteps(pcg_precond, num_CR_relax_steps);
+         HYPRE_BoomerAMGSetCRRate(pcg_precond, CR_rate);
+         HYPRE_BoomerAMGSetCRStrongTh(pcg_precond, CR_strong_th);
+         HYPRE_BoomerAMGSetCRUseCG(pcg_precond, CR_use_CG);
+         if (relax_type > -1) HYPRE_BoomerAMGSetRelaxType(pcg_precond, relax_type);
+         if (relax_down > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_down, 1);
+         if (relax_up > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_up, 2);
+         if (relax_coarse > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_coarse, 3);
+         HYPRE_BoomerAMGSetAddRelaxType(pcg_precond, add_relax_type);
+         HYPRE_BoomerAMGSetAddRelaxWt(pcg_precond, add_relax_wt);
+         HYPRE_BoomerAMGSetChebyOrder(pcg_precond, cheby_order);
+         HYPRE_BoomerAMGSetChebyFraction(pcg_precond, cheby_fraction);
+         HYPRE_BoomerAMGSetRelaxOrder(pcg_precond, relax_order);
+         HYPRE_BoomerAMGSetRelaxWt(pcg_precond, relax_wt);
+         HYPRE_BoomerAMGSetOuterWt(pcg_precond, outer_wt);
+         if (level_w > -1)
+            HYPRE_BoomerAMGSetLevelRelaxWt(pcg_precond, relax_wt_level,level_w);
+         if (level_ow > -1)
+            HYPRE_BoomerAMGSetLevelOuterWt(pcg_precond,outer_wt_level,level_ow);
          HYPRE_BoomerAMGSetSmoothType(pcg_precond, smooth_type);
          HYPRE_BoomerAMGSetSmoothNumLevels(pcg_precond, smooth_num_levels);
          HYPRE_BoomerAMGSetSmoothNumSweeps(pcg_precond, smooth_num_sweeps);
          HYPRE_BoomerAMGSetVariant(pcg_precond, variant);
          HYPRE_BoomerAMGSetOverlap(pcg_precond, overlap);
          HYPRE_BoomerAMGSetDomainType(pcg_precond, domain_type);
+         HYPRE_BoomerAMGSetSchwarzUseNonSymm(pcg_precond, use_nonsymm_schwarz);
          HYPRE_BoomerAMGSetSchwarzRlxWeight(pcg_precond, schwarz_rlx_weight);
-         HYPRE_BoomerAMGSetGridRelaxPoints(pcg_precond, grid_relax_points);
+         if (eu_level < 0) eu_level = 0;
+         HYPRE_BoomerAMGSetEuLevel(pcg_precond, eu_level);
+         HYPRE_BoomerAMGSetEuBJ(pcg_precond, eu_bj);
+         HYPRE_BoomerAMGSetEuSparseA(pcg_precond, eu_sparse_A);
          HYPRE_BoomerAMGSetMaxLevels(pcg_precond, max_levels);
          HYPRE_BoomerAMGSetMaxRowSum(pcg_precond, max_row_sum);
+         HYPRE_BoomerAMGSetDebugFlag(pcg_precond, debug_flag);
          HYPRE_BoomerAMGSetNumFunctions(pcg_precond, num_functions);
-         HYPRE_BoomerAMGSetNumPaths(pcg_precond, num_paths);
          HYPRE_BoomerAMGSetAggNumLevels(pcg_precond, agg_num_levels);
+         HYPRE_BoomerAMGSetAggInterpType(pcg_precond, agg_interp_type);
+         HYPRE_BoomerAMGSetAggTruncFactor(pcg_precond, agg_trunc_factor);
+         HYPRE_BoomerAMGSetAggP12TruncFactor(pcg_precond, agg_P12_trunc_factor);
+         HYPRE_BoomerAMGSetAggPMaxElmts(pcg_precond, agg_P_max_elmts);
+         HYPRE_BoomerAMGSetAggP12MaxElmts(pcg_precond, agg_P12_max_elmts);
+         HYPRE_BoomerAMGSetNumPaths(pcg_precond, num_paths);
+         HYPRE_BoomerAMGSetNodal(pcg_precond, nodal);
+         HYPRE_BoomerAMGSetNodalDiag(pcg_precond, nodal_diag);
+         HYPRE_BoomerAMGSetCycleNumSweeps(pcg_precond, ns_coarse, 3);
          if (num_functions > 1)
             HYPRE_BoomerAMGSetDofFunc(pcg_precond, dof_func);
+         HYPRE_BoomerAMGSetAdditive(pcg_precond, additive);
+         HYPRE_BoomerAMGSetMultAdditive(pcg_precond, mult_add);
+         HYPRE_BoomerAMGSetSimple(pcg_precond, simple);
+         HYPRE_BoomerAMGSetMultAddPMaxElmts(pcg_precond, add_P_max_elmts);
+         HYPRE_BoomerAMGSetMultAddTruncFactor(pcg_precond, add_trunc_factor);
+         HYPRE_BoomerAMGSetRAP2(pcg_precond, rap2);
+         HYPRE_BoomerAMGSetKeepTranspose(pcg_precond, keepTranspose);
+         if (nongalerk_tol)
+         {
+            HYPRE_BoomerAMGSetNonGalerkinTol(pcg_precond, nongalerk_tol[nongalerk_num_tol-1]);
+            for (i=0; i < nongalerk_num_tol-1; i++)
+               HYPRE_BoomerAMGSetLevelNonGalerkinTol(pcg_precond, nongalerk_tol[i], i);
+         }
+         HYPRE_GMRESSetMaxIter(pcg_solver, mg_max_iter);
          HYPRE_GMRESSetPrecond(pcg_solver,
-                             (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
-                             (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
-                                   pcg_precond);
+                               (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
+                               (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
+                               pcg_precond);
       }
       else if (solver_id == 18)
       {
@@ -3467,12 +4280,14 @@ main( hypre_int argc,
 
          HYPRE_EuclidCreate(hypre_MPI_COMM_WORLD, &pcg_precond);
 
-         /* note: There are three three methods of setting run-time 
-            parameters for Euclid: (see HYPRE_parcsr_ls.h); here
-            we'll use what I think is simplest: let Euclid internally 
-            parse the command line.
-         */   
-         HYPRE_EuclidSetParams(pcg_precond, argc, argv);
+         if (eu_level > -1) HYPRE_EuclidSetLevel(pcg_precond, eu_level);
+         if (eu_ilut) HYPRE_EuclidSetILUT(pcg_precond, eu_ilut);
+         if (eu_sparse_A) HYPRE_EuclidSetSparseA(pcg_precond, eu_sparse_A);
+         if (eu_row_scale) HYPRE_EuclidSetRowScale(pcg_precond, eu_row_scale);
+         if (eu_bj) HYPRE_EuclidSetBJ(pcg_precond, eu_bj);
+         HYPRE_EuclidSetStats(pcg_precond, eu_stats);
+         HYPRE_EuclidSetMem(pcg_precond, eu_mem);
+         /*HYPRE_EuclidSetParams(pcg_precond, argc, argv);*/
 
          HYPRE_GMRESSetPrecond (pcg_solver,
                                 (HYPRE_PtrToSolverFcn) HYPRE_EuclidSolve,
@@ -3483,12 +4298,12 @@ main( hypre_int argc,
       HYPRE_GMRESGetPrecond(pcg_solver, &pcg_precond_gotten);
       if (pcg_precond_gotten != pcg_precond)
       {
-        hypre_printf("HYPRE_GMRESGetPrecond got bad precond\n");
-        return(-1);
+         hypre_printf("HYPRE_GMRESGetPrecond got bad precond\n");
+         return(-1);
       }
       else
-        if (myid == 0)
-          hypre_printf("HYPRE_GMRESGetPrecond got good precond\n");
+         if (myid == 0)
+            hypre_printf("HYPRE_GMRESGetPrecond got good precond\n");
       HYPRE_GMRESSetup
          (pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, (HYPRE_Vector)x);
  
@@ -3514,9 +4329,9 @@ main( hypre_int argc,
       /* run a second time to check for memory leaks */
       HYPRE_ParVectorSetRandomValues(x, 775);
       HYPRE_GMRESSetup(pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, 
-		(HYPRE_Vector)x);
+                       (HYPRE_Vector)x);
       HYPRE_GMRESSolve(pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, 
-		(HYPRE_Vector)x);
+                       (HYPRE_Vector)x);
 #endif
 
       HYPRE_ParCSRGMRESDestroy(pcg_solver);
@@ -3536,7 +4351,7 @@ main( hypre_int argc,
       }
       else if (solver_id == 44)
       {
-        HYPRE_EuclidDestroy(pcg_precond);
+         HYPRE_EuclidDestroy(pcg_precond);
       }
 
       if (myid == 0)
@@ -3547,6 +4362,360 @@ main( hypre_int argc,
          hypre_printf("\n");
       }
    }
+/*-----------------------------------------------------------
+ * Solve the system using LGMRES 
+ *-----------------------------------------------------------*/
+
+   if (solver_id == 50 || solver_id == 51 )
+   {
+      time_index = hypre_InitializeTiming("LGMRES Setup");
+      hypre_BeginTiming(time_index);
+ 
+      HYPRE_ParCSRLGMRESCreate(hypre_MPI_COMM_WORLD, &pcg_solver);
+      HYPRE_LGMRESSetKDim(pcg_solver, k_dim);
+      HYPRE_LGMRESSetAugDim(pcg_solver, aug_dim);
+      HYPRE_LGMRESSetMaxIter(pcg_solver, max_iter);
+      HYPRE_LGMRESSetTol(pcg_solver, tol);
+      HYPRE_LGMRESSetAbsoluteTol(pcg_solver, atol);
+      HYPRE_LGMRESSetLogging(pcg_solver, 1);
+      HYPRE_LGMRESSetPrintLevel(pcg_solver, ioutdat);
+ 
+      if (solver_id == 51)
+      {
+         /* use BoomerAMG as preconditioner */
+         if (myid == 0) hypre_printf("Solver: AMG-LGMRES\n");
+
+         HYPRE_BoomerAMGCreate(&pcg_precond); 
+         HYPRE_BoomerAMGSetCGCIts(pcg_precond, cgcits);
+         HYPRE_BoomerAMGSetInterpType(pcg_precond, interp_type);
+         HYPRE_BoomerAMGSetPostInterpType(pcg_precond, post_interp_type);
+         HYPRE_BoomerAMGSetNumSamples(pcg_precond, gsmg_samples);
+         HYPRE_BoomerAMGSetTol(pcg_precond, pc_tol);
+         HYPRE_BoomerAMGSetCoarsenType(pcg_precond, coarsen_type);
+         HYPRE_BoomerAMGSetMeasureType(pcg_precond, measure_type);
+         HYPRE_BoomerAMGSetStrongThreshold(pcg_precond, strong_threshold);
+         HYPRE_BoomerAMGSetSeqThreshold(pcg_precond, seq_threshold);
+         HYPRE_BoomerAMGSetRedundant(pcg_precond, redundant);
+         HYPRE_BoomerAMGSetMaxCoarseSize(pcg_precond, coarse_threshold);
+         HYPRE_BoomerAMGSetMinCoarseSize(pcg_precond, min_coarse_size);
+         HYPRE_BoomerAMGSetTruncFactor(pcg_precond, trunc_factor);
+         HYPRE_BoomerAMGSetPMaxElmts(pcg_precond, P_max_elmts);
+         HYPRE_BoomerAMGSetJacobiTruncThreshold(pcg_precond, jacobi_trunc_threshold);
+         HYPRE_BoomerAMGSetSCommPkgSwitch(pcg_precond, S_commpkg_switch);
+         HYPRE_BoomerAMGSetPrintLevel(pcg_precond, poutdat);
+         HYPRE_BoomerAMGSetPrintFileName(pcg_precond, "driver.out.log");
+         HYPRE_BoomerAMGSetMaxIter(pcg_precond, 1);
+         HYPRE_BoomerAMGSetCycleType(pcg_precond, cycle_type);
+         HYPRE_BoomerAMGSetNumSweeps(pcg_precond, num_sweeps);
+         HYPRE_BoomerAMGSetISType(pcg_precond, IS_type);
+         HYPRE_BoomerAMGSetNumCRRelaxSteps(pcg_precond, num_CR_relax_steps);
+         HYPRE_BoomerAMGSetCRRate(pcg_precond, CR_rate);
+         HYPRE_BoomerAMGSetCRStrongTh(pcg_precond, CR_strong_th);
+         HYPRE_BoomerAMGSetCRUseCG(pcg_precond, CR_use_CG);
+         if (relax_type > -1) HYPRE_BoomerAMGSetRelaxType(pcg_precond, relax_type);
+         if (relax_down > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_down, 1);
+         if (relax_up > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_up, 2);
+         if (relax_coarse > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_coarse, 3);
+         HYPRE_BoomerAMGSetAddRelaxType(pcg_precond, add_relax_type);
+         HYPRE_BoomerAMGSetAddRelaxWt(pcg_precond, add_relax_wt);
+         HYPRE_BoomerAMGSetChebyOrder(pcg_precond, cheby_order);
+         HYPRE_BoomerAMGSetChebyFraction(pcg_precond, cheby_fraction);
+         HYPRE_BoomerAMGSetRelaxOrder(pcg_precond, relax_order);
+         HYPRE_BoomerAMGSetRelaxWt(pcg_precond, relax_wt);
+         HYPRE_BoomerAMGSetOuterWt(pcg_precond, outer_wt);
+         if (level_w > -1)
+            HYPRE_BoomerAMGSetLevelRelaxWt(pcg_precond, relax_wt_level,level_w);
+         if (level_ow > -1)
+            HYPRE_BoomerAMGSetLevelOuterWt(pcg_precond,outer_wt_level,level_ow);
+         HYPRE_BoomerAMGSetSmoothType(pcg_precond, smooth_type);
+         HYPRE_BoomerAMGSetSmoothNumLevels(pcg_precond, smooth_num_levels);
+         HYPRE_BoomerAMGSetSmoothNumSweeps(pcg_precond, smooth_num_sweeps);
+         HYPRE_BoomerAMGSetMaxLevels(pcg_precond, max_levels);
+         HYPRE_BoomerAMGSetMaxRowSum(pcg_precond, max_row_sum);
+         HYPRE_BoomerAMGSetDebugFlag(pcg_precond, debug_flag);
+         HYPRE_BoomerAMGSetNumFunctions(pcg_precond, num_functions);
+         HYPRE_BoomerAMGSetAggNumLevels(pcg_precond, agg_num_levels);
+         HYPRE_BoomerAMGSetAggInterpType(pcg_precond, agg_interp_type);
+         HYPRE_BoomerAMGSetAggTruncFactor(pcg_precond, agg_trunc_factor);
+         HYPRE_BoomerAMGSetAggP12TruncFactor(pcg_precond, agg_P12_trunc_factor);
+         HYPRE_BoomerAMGSetAggPMaxElmts(pcg_precond, agg_P_max_elmts);
+         HYPRE_BoomerAMGSetAggP12MaxElmts(pcg_precond, agg_P12_max_elmts);
+         HYPRE_BoomerAMGSetNumPaths(pcg_precond, num_paths);
+         HYPRE_BoomerAMGSetNodal(pcg_precond, nodal);
+         HYPRE_BoomerAMGSetNodalDiag(pcg_precond, nodal_diag);
+         HYPRE_BoomerAMGSetVariant(pcg_precond, variant);
+         HYPRE_BoomerAMGSetOverlap(pcg_precond, overlap);
+         HYPRE_BoomerAMGSetDomainType(pcg_precond, domain_type);
+         HYPRE_BoomerAMGSetSchwarzUseNonSymm(pcg_precond, use_nonsymm_schwarz);
+         HYPRE_BoomerAMGSetSchwarzRlxWeight(pcg_precond, schwarz_rlx_weight);
+         if (eu_level < 0) eu_level = 0;
+         HYPRE_BoomerAMGSetEuLevel(pcg_precond, eu_level);
+         HYPRE_BoomerAMGSetEuBJ(pcg_precond, eu_bj);
+         HYPRE_BoomerAMGSetEuSparseA(pcg_precond, eu_sparse_A);
+         HYPRE_BoomerAMGSetCycleNumSweeps(pcg_precond, ns_coarse, 3);
+         if (num_functions > 1)
+            HYPRE_BoomerAMGSetDofFunc(pcg_precond, dof_func);
+         HYPRE_BoomerAMGSetAdditive(pcg_precond, additive);
+         HYPRE_BoomerAMGSetMultAdditive(pcg_precond, mult_add);
+         HYPRE_BoomerAMGSetSimple(pcg_precond, simple);
+         HYPRE_BoomerAMGSetMultAddPMaxElmts(pcg_precond, add_P_max_elmts);
+         HYPRE_BoomerAMGSetMultAddTruncFactor(pcg_precond, add_trunc_factor);
+         HYPRE_BoomerAMGSetRAP2(pcg_precond, rap2);
+         HYPRE_BoomerAMGSetKeepTranspose(pcg_precond, keepTranspose);
+         if (nongalerk_tol)
+         {
+            HYPRE_BoomerAMGSetNonGalerkinTol(pcg_precond, nongalerk_tol[nongalerk_num_tol-1]);
+            for (i=0; i < nongalerk_num_tol-1; i++)
+               HYPRE_BoomerAMGSetLevelNonGalerkinTol(pcg_precond, nongalerk_tol[i], i);
+         }
+         HYPRE_LGMRESSetMaxIter(pcg_solver, mg_max_iter);
+         HYPRE_LGMRESSetPrecond(pcg_solver,
+                                (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
+                                (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
+                                pcg_precond);
+      }
+      else if (solver_id == 50)
+      {
+         /* use diagonal scaling as preconditioner */
+         if (myid == 0) hypre_printf("Solver: DS-LGMRES\n");
+         pcg_precond = NULL;
+
+         HYPRE_LGMRESSetPrecond(pcg_solver,
+                                (HYPRE_PtrToSolverFcn) HYPRE_ParCSRDiagScale,
+                                (HYPRE_PtrToSolverFcn) HYPRE_ParCSRDiagScaleSetup,
+                                pcg_precond);
+      }
+
+      HYPRE_LGMRESGetPrecond(pcg_solver, &pcg_precond_gotten);
+      if (pcg_precond_gotten != pcg_precond)
+      {
+         hypre_printf("HYPRE_LGMRESGetPrecond got bad precond\n");
+         return(-1);
+      }
+      else
+         if (myid == 0)
+            hypre_printf("HYPRE_LGMRESGetPrecond got good precond\n");
+      HYPRE_LGMRESSetup
+         (pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, (HYPRE_Vector)x);
+ 
+      hypre_EndTiming(time_index);
+      hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
+      hypre_FinalizeTiming(time_index);
+      hypre_ClearTiming();
+   
+      time_index = hypre_InitializeTiming("LGMRES Solve");
+      hypre_BeginTiming(time_index);
+ 
+      HYPRE_LGMRESSolve
+         (pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, (HYPRE_Vector)x);
+ 
+      hypre_EndTiming(time_index);
+      hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
+      hypre_FinalizeTiming(time_index);
+      hypre_ClearTiming();
+ 
+      HYPRE_LGMRESGetNumIterations(pcg_solver, &num_iterations);
+      HYPRE_LGMRESGetFinalRelativeResidualNorm(pcg_solver,&final_res_norm);
+
+      HYPRE_ParCSRLGMRESDestroy(pcg_solver);
+ 
+      if (solver_id == 51)
+      {
+         HYPRE_BoomerAMGDestroy(pcg_precond);
+      }
+
+      if (myid == 0)
+      {
+         hypre_printf("\n");
+         hypre_printf("LGMRES Iterations = %d\n", num_iterations);
+         hypre_printf("Final LGMRES Relative Residual Norm = %e\n", final_res_norm);
+         hypre_printf("\n");
+      }
+   }
+
+   /*-----------------------------------------------------------
+    * Solve the system using FlexGMRES 
+    *-----------------------------------------------------------*/
+
+   if (solver_id == 60 || solver_id == 61 )
+   {
+      time_index = hypre_InitializeTiming("FlexGMRES Setup");
+      hypre_BeginTiming(time_index);
+ 
+      HYPRE_ParCSRFlexGMRESCreate(hypre_MPI_COMM_WORLD, &pcg_solver);
+      HYPRE_FlexGMRESSetKDim(pcg_solver, k_dim);
+      HYPRE_FlexGMRESSetMaxIter(pcg_solver, max_iter);
+      HYPRE_FlexGMRESSetTol(pcg_solver, tol);
+      HYPRE_FlexGMRESSetAbsoluteTol(pcg_solver, atol);
+      HYPRE_FlexGMRESSetLogging(pcg_solver, 1);
+      HYPRE_FlexGMRESSetPrintLevel(pcg_solver, ioutdat);
+ 
+      if (solver_id == 61)
+      {
+         /* use BoomerAMG as preconditioner */
+         if (myid == 0) hypre_printf("Solver: AMG-FlexGMRES\n");
+
+         HYPRE_BoomerAMGCreate(&pcg_precond); 
+         HYPRE_BoomerAMGSetCGCIts(pcg_precond, cgcits);
+         HYPRE_BoomerAMGSetInterpType(pcg_precond, interp_type);
+         HYPRE_BoomerAMGSetPostInterpType(pcg_precond, post_interp_type);
+         HYPRE_BoomerAMGSetNumSamples(pcg_precond, gsmg_samples);
+         HYPRE_BoomerAMGSetTol(pcg_precond, pc_tol);
+         HYPRE_BoomerAMGSetCoarsenType(pcg_precond, coarsen_type);
+         HYPRE_BoomerAMGSetMeasureType(pcg_precond, measure_type);
+         HYPRE_BoomerAMGSetStrongThreshold(pcg_precond, strong_threshold);
+         HYPRE_BoomerAMGSetSeqThreshold(pcg_precond, seq_threshold);
+         HYPRE_BoomerAMGSetRedundant(pcg_precond, redundant);
+         HYPRE_BoomerAMGSetMaxCoarseSize(pcg_precond, coarse_threshold);
+         HYPRE_BoomerAMGSetMinCoarseSize(pcg_precond, min_coarse_size);
+         HYPRE_BoomerAMGSetTruncFactor(pcg_precond, trunc_factor);
+         HYPRE_BoomerAMGSetPMaxElmts(pcg_precond, P_max_elmts);
+         HYPRE_BoomerAMGSetJacobiTruncThreshold(pcg_precond, jacobi_trunc_threshold);
+         HYPRE_BoomerAMGSetSCommPkgSwitch(pcg_precond, S_commpkg_switch);
+         HYPRE_BoomerAMGSetPrintLevel(pcg_precond, poutdat);
+         HYPRE_BoomerAMGSetPrintFileName(pcg_precond, "driver.out.log");
+         HYPRE_BoomerAMGSetMaxIter(pcg_precond, 1);
+         HYPRE_BoomerAMGSetCycleType(pcg_precond, cycle_type);
+         HYPRE_BoomerAMGSetNumSweeps(pcg_precond, num_sweeps);
+         HYPRE_BoomerAMGSetISType(pcg_precond, IS_type);
+         HYPRE_BoomerAMGSetNumCRRelaxSteps(pcg_precond, num_CR_relax_steps);
+         HYPRE_BoomerAMGSetCRRate(pcg_precond, CR_rate);
+         HYPRE_BoomerAMGSetCRStrongTh(pcg_precond, CR_strong_th);
+         HYPRE_BoomerAMGSetCRUseCG(pcg_precond, CR_use_CG);
+         if (relax_type > -1) HYPRE_BoomerAMGSetRelaxType(pcg_precond, relax_type);
+         if (relax_down > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_down, 1);
+         if (relax_up > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_up, 2);
+         if (relax_coarse > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_coarse, 3);
+         HYPRE_BoomerAMGSetAddRelaxType(pcg_precond, add_relax_type);
+         HYPRE_BoomerAMGSetAddRelaxWt(pcg_precond, add_relax_wt);
+         HYPRE_BoomerAMGSetChebyOrder(pcg_precond, cheby_order);
+         HYPRE_BoomerAMGSetChebyFraction(pcg_precond, cheby_fraction);
+         HYPRE_BoomerAMGSetRelaxOrder(pcg_precond, relax_order);
+         HYPRE_BoomerAMGSetRelaxWt(pcg_precond, relax_wt);
+         HYPRE_BoomerAMGSetOuterWt(pcg_precond, outer_wt);
+         if (level_w > -1)
+            HYPRE_BoomerAMGSetLevelRelaxWt(pcg_precond, relax_wt_level,level_w);
+         if (level_ow > -1)
+            HYPRE_BoomerAMGSetLevelOuterWt(pcg_precond,outer_wt_level,level_ow);
+         HYPRE_BoomerAMGSetSmoothType(pcg_precond, smooth_type);
+         HYPRE_BoomerAMGSetSmoothNumLevels(pcg_precond, smooth_num_levels);
+         HYPRE_BoomerAMGSetSmoothNumSweeps(pcg_precond, smooth_num_sweeps);
+         HYPRE_BoomerAMGSetMaxLevels(pcg_precond, max_levels);
+         HYPRE_BoomerAMGSetMaxRowSum(pcg_precond, max_row_sum);
+         HYPRE_BoomerAMGSetDebugFlag(pcg_precond, debug_flag);
+         HYPRE_BoomerAMGSetNumFunctions(pcg_precond, num_functions);
+         HYPRE_BoomerAMGSetAggNumLevels(pcg_precond, agg_num_levels);
+         HYPRE_BoomerAMGSetAggInterpType(pcg_precond, agg_interp_type);
+         HYPRE_BoomerAMGSetAggTruncFactor(pcg_precond, agg_trunc_factor);
+         HYPRE_BoomerAMGSetAggP12TruncFactor(pcg_precond, agg_P12_trunc_factor);
+         HYPRE_BoomerAMGSetAggPMaxElmts(pcg_precond, agg_P_max_elmts);
+         HYPRE_BoomerAMGSetAggP12MaxElmts(pcg_precond, agg_P12_max_elmts);
+         HYPRE_BoomerAMGSetNumPaths(pcg_precond, num_paths);
+         HYPRE_BoomerAMGSetNodal(pcg_precond, nodal);
+         HYPRE_BoomerAMGSetNodalDiag(pcg_precond, nodal_diag);
+         HYPRE_BoomerAMGSetVariant(pcg_precond, variant);
+         HYPRE_BoomerAMGSetOverlap(pcg_precond, overlap);
+         HYPRE_BoomerAMGSetDomainType(pcg_precond, domain_type);
+         HYPRE_BoomerAMGSetSchwarzUseNonSymm(pcg_precond, use_nonsymm_schwarz);
+         HYPRE_BoomerAMGSetSchwarzRlxWeight(pcg_precond, schwarz_rlx_weight);
+         if (eu_level < 0) eu_level = 0;
+         HYPRE_BoomerAMGSetEuLevel(pcg_precond, eu_level);
+         HYPRE_BoomerAMGSetEuBJ(pcg_precond, eu_bj);
+         HYPRE_BoomerAMGSetEuSparseA(pcg_precond, eu_sparse_A);
+         HYPRE_BoomerAMGSetCycleNumSweeps(pcg_precond, ns_coarse, 3);
+         if (num_functions > 1)
+            HYPRE_BoomerAMGSetDofFunc(pcg_precond, dof_func);
+         HYPRE_BoomerAMGSetAdditive(pcg_precond, additive);
+         HYPRE_BoomerAMGSetMultAdditive(pcg_precond, mult_add);
+         HYPRE_BoomerAMGSetSimple(pcg_precond, simple);
+         HYPRE_BoomerAMGSetMultAddPMaxElmts(pcg_precond, add_P_max_elmts);
+         HYPRE_BoomerAMGSetMultAddTruncFactor(pcg_precond, add_trunc_factor);
+         HYPRE_BoomerAMGSetRAP2(pcg_precond, rap2);
+         HYPRE_BoomerAMGSetKeepTranspose(pcg_precond, keepTranspose);
+         if (nongalerk_tol)
+         {
+            HYPRE_BoomerAMGSetNonGalerkinTol(pcg_precond, nongalerk_tol[nongalerk_num_tol-1]);
+            for (i=0; i < nongalerk_num_tol-1; i++)
+               HYPRE_BoomerAMGSetLevelNonGalerkinTol(pcg_precond, nongalerk_tol[i], i);
+         }
+         HYPRE_FlexGMRESSetMaxIter(pcg_solver, mg_max_iter);
+         HYPRE_FlexGMRESSetPrecond(pcg_solver,
+                                   (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
+                                   (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
+                                   pcg_precond);
+      }
+      else if (solver_id == 60)
+      {
+         /* use diagonal scaling as preconditioner */
+         if (myid == 0) hypre_printf("Solver: DS-FlexGMRES\n");
+         pcg_precond = NULL;
+
+         HYPRE_FlexGMRESSetPrecond(pcg_solver,
+                                   (HYPRE_PtrToSolverFcn) HYPRE_ParCSRDiagScale,
+                                   (HYPRE_PtrToSolverFcn) HYPRE_ParCSRDiagScaleSetup,
+                                   pcg_precond);
+      }
+
+      HYPRE_FlexGMRESGetPrecond(pcg_solver, &pcg_precond_gotten);
+      if (pcg_precond_gotten != pcg_precond)
+      {
+         hypre_printf("HYPRE_FlexGMRESGetPrecond got bad precond\n");
+         return(-1);
+      }
+      else
+         if (myid == 0)
+            hypre_printf("HYPRE_FlexGMRESGetPrecond got good precond\n");
+
+
+      /* this is optional - could be a user defined one instead (see ex5.c)*/
+      HYPRE_FlexGMRESSetModifyPC( pcg_solver, 
+                                  (HYPRE_PtrToModifyPCFcn) hypre_FlexGMRESModifyPCDefault);
+
+
+      HYPRE_FlexGMRESSetup
+         (pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, (HYPRE_Vector)x);
+ 
+      hypre_EndTiming(time_index);
+      hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
+      hypre_FinalizeTiming(time_index);
+      hypre_ClearTiming();
+   
+      time_index = hypre_InitializeTiming("FlexGMRES Solve");
+      hypre_BeginTiming(time_index);
+ 
+      HYPRE_FlexGMRESSolve
+         (pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, (HYPRE_Vector)x);
+ 
+      hypre_EndTiming(time_index);
+      hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
+      hypre_FinalizeTiming(time_index);
+      hypre_ClearTiming();
+ 
+      HYPRE_FlexGMRESGetNumIterations(pcg_solver, &num_iterations);
+      HYPRE_FlexGMRESGetFinalRelativeResidualNorm(pcg_solver,&final_res_norm);
+
+      HYPRE_ParCSRFlexGMRESDestroy(pcg_solver);
+ 
+      if (solver_id == 61)
+      {
+         HYPRE_BoomerAMGDestroy(pcg_precond);
+      }
+
+      if (myid == 0)
+      {
+         hypre_printf("\n");
+         hypre_printf("FlexGMRES Iterations = %d\n", num_iterations);
+         hypre_printf("Final FlexGMRES Relative Residual Norm = %e\n", final_res_norm);
+         hypre_printf("\n");
+      }
+   }
+
    /*-----------------------------------------------------------
     * Solve the system using BiCGSTAB 
     *-----------------------------------------------------------*/
@@ -3557,8 +4726,9 @@ main( hypre_int argc,
       hypre_BeginTiming(time_index);
  
       HYPRE_ParCSRBiCGSTABCreate(hypre_MPI_COMM_WORLD, &pcg_solver);
-      HYPRE_BiCGSTABSetMaxIter(pcg_solver, 1000);
+      HYPRE_BiCGSTABSetMaxIter(pcg_solver, max_iter);
       HYPRE_BiCGSTABSetTol(pcg_solver, tol);
+      HYPRE_BiCGSTABSetAbsoluteTol(pcg_solver, atol);
       HYPRE_BiCGSTABSetLogging(pcg_solver, ioutdat);
       HYPRE_BiCGSTABSetPrintLevel(pcg_solver, ioutdat);
  
@@ -3567,40 +4737,93 @@ main( hypre_int argc,
          /* use BoomerAMG as preconditioner */
          if (myid == 0) hypre_printf("Solver: AMG-BiCGSTAB\n");
          HYPRE_BoomerAMGCreate(&pcg_precond); 
-         /* BM Aug 25, 2006 */
          HYPRE_BoomerAMGSetCGCIts(pcg_precond, cgcits);
          HYPRE_BoomerAMGSetInterpType(pcg_precond, interp_type);
+         HYPRE_BoomerAMGSetPostInterpType(pcg_precond, post_interp_type);
          HYPRE_BoomerAMGSetNumSamples(pcg_precond, gsmg_samples);
          HYPRE_BoomerAMGSetTol(pcg_precond, pc_tol);
-         HYPRE_BoomerAMGSetCoarsenType(pcg_precond, (hybrid*coarsen_type));
+         HYPRE_BoomerAMGSetCoarsenType(pcg_precond, coarsen_type);
          HYPRE_BoomerAMGSetMeasureType(pcg_precond, measure_type);
          HYPRE_BoomerAMGSetStrongThreshold(pcg_precond, strong_threshold);
+         HYPRE_BoomerAMGSetSeqThreshold(pcg_precond, seq_threshold);
+         HYPRE_BoomerAMGSetRedundant(pcg_precond, redundant);
+         HYPRE_BoomerAMGSetMaxCoarseSize(pcg_precond, coarse_threshold);
+         HYPRE_BoomerAMGSetMinCoarseSize(pcg_precond, min_coarse_size);
          HYPRE_BoomerAMGSetTruncFactor(pcg_precond, trunc_factor);
          HYPRE_BoomerAMGSetPMaxElmts(pcg_precond, P_max_elmts);
-         HYPRE_BoomerAMGSetPostInterpType(pcg_precond, post_interp_type);
          HYPRE_BoomerAMGSetJacobiTruncThreshold(pcg_precond, jacobi_trunc_threshold);
+         HYPRE_BoomerAMGSetSCommPkgSwitch(pcg_precond, S_commpkg_switch);
          HYPRE_BoomerAMGSetPrintLevel(pcg_precond, poutdat);
          HYPRE_BoomerAMGSetPrintFileName(pcg_precond, "driver.out.log");
          HYPRE_BoomerAMGSetMaxIter(pcg_precond, 1);
          HYPRE_BoomerAMGSetCycleType(pcg_precond, cycle_type);
-         HYPRE_BoomerAMGSetNumGridSweeps(pcg_precond, num_grid_sweeps);
-         HYPRE_BoomerAMGSetGridRelaxType(pcg_precond, grid_relax_type);
-         HYPRE_BoomerAMGSetRelaxWeight(pcg_precond, relax_weight);
-         HYPRE_BoomerAMGSetOmega(pcg_precond, omega);
+         HYPRE_BoomerAMGSetNumSweeps(pcg_precond, num_sweeps);
+         HYPRE_BoomerAMGSetISType(pcg_precond, IS_type);
+         HYPRE_BoomerAMGSetNumCRRelaxSteps(pcg_precond, num_CR_relax_steps);
+         HYPRE_BoomerAMGSetCRRate(pcg_precond, CR_rate);
+         HYPRE_BoomerAMGSetCRStrongTh(pcg_precond, CR_strong_th);
+         HYPRE_BoomerAMGSetCRUseCG(pcg_precond, CR_use_CG);
+         if (relax_type > -1) HYPRE_BoomerAMGSetRelaxType(pcg_precond, relax_type);
+         if (relax_down > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_down, 1);
+         if (relax_up > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_up, 2);
+         if (relax_coarse > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_coarse, 3);
+         HYPRE_BoomerAMGSetAddRelaxType(pcg_precond, add_relax_type);
+         HYPRE_BoomerAMGSetAddRelaxWt(pcg_precond, add_relax_wt);
+         HYPRE_BoomerAMGSetChebyOrder(pcg_precond, cheby_order);
+         HYPRE_BoomerAMGSetChebyFraction(pcg_precond, cheby_fraction);
+         HYPRE_BoomerAMGSetRelaxOrder(pcg_precond, relax_order);
+         HYPRE_BoomerAMGSetRelaxWt(pcg_precond, relax_wt);
+         HYPRE_BoomerAMGSetOuterWt(pcg_precond, outer_wt);
+         if (level_w > -1)
+            HYPRE_BoomerAMGSetLevelRelaxWt(pcg_precond, relax_wt_level,level_w);
+         if (level_ow > -1)
+            HYPRE_BoomerAMGSetLevelOuterWt(pcg_precond,outer_wt_level,level_ow);
          HYPRE_BoomerAMGSetSmoothType(pcg_precond, smooth_type);
          HYPRE_BoomerAMGSetSmoothNumLevels(pcg_precond, smooth_num_levels);
          HYPRE_BoomerAMGSetSmoothNumSweeps(pcg_precond, smooth_num_sweeps);
-         HYPRE_BoomerAMGSetGridRelaxPoints(pcg_precond, grid_relax_points);
          HYPRE_BoomerAMGSetMaxLevels(pcg_precond, max_levels);
          HYPRE_BoomerAMGSetMaxRowSum(pcg_precond, max_row_sum);
+         HYPRE_BoomerAMGSetDebugFlag(pcg_precond, debug_flag);
          HYPRE_BoomerAMGSetNumFunctions(pcg_precond, num_functions);
-         HYPRE_BoomerAMGSetNumPaths(pcg_precond, num_paths);
          HYPRE_BoomerAMGSetAggNumLevels(pcg_precond, agg_num_levels);
+         HYPRE_BoomerAMGSetAggInterpType(pcg_precond, agg_interp_type);
+         HYPRE_BoomerAMGSetAggTruncFactor(pcg_precond, agg_trunc_factor);
+         HYPRE_BoomerAMGSetAggP12TruncFactor(pcg_precond, agg_P12_trunc_factor);
+         HYPRE_BoomerAMGSetAggPMaxElmts(pcg_precond, agg_P_max_elmts);
+         HYPRE_BoomerAMGSetAggP12MaxElmts(pcg_precond, agg_P12_max_elmts);
+         HYPRE_BoomerAMGSetNumPaths(pcg_precond, num_paths);
+         HYPRE_BoomerAMGSetNodal(pcg_precond, nodal);
+         HYPRE_BoomerAMGSetNodalDiag(pcg_precond, nodal_diag);
          HYPRE_BoomerAMGSetVariant(pcg_precond, variant);
          HYPRE_BoomerAMGSetOverlap(pcg_precond, overlap);
          HYPRE_BoomerAMGSetDomainType(pcg_precond, domain_type);
+         HYPRE_BoomerAMGSetSchwarzUseNonSymm(pcg_precond, use_nonsymm_schwarz);
+       
+         HYPRE_BoomerAMGSetSchwarzRlxWeight(pcg_precond, schwarz_rlx_weight);
+         if (eu_level < 0) eu_level = 0;
+         HYPRE_BoomerAMGSetEuLevel(pcg_precond, eu_level);
+         HYPRE_BoomerAMGSetEuBJ(pcg_precond, eu_bj);
+         HYPRE_BoomerAMGSetEuSparseA(pcg_precond, eu_sparse_A);
+         HYPRE_BoomerAMGSetCycleNumSweeps(pcg_precond, ns_coarse, 3);
          if (num_functions > 1)
             HYPRE_BoomerAMGSetDofFunc(pcg_precond, dof_func);
+         HYPRE_BoomerAMGSetAdditive(pcg_precond, additive);
+         HYPRE_BoomerAMGSetMultAdditive(pcg_precond, mult_add);
+         HYPRE_BoomerAMGSetSimple(pcg_precond, simple);
+         HYPRE_BoomerAMGSetMultAddPMaxElmts(pcg_precond, add_P_max_elmts);
+         HYPRE_BoomerAMGSetMultAddTruncFactor(pcg_precond, add_trunc_factor);
+         HYPRE_BoomerAMGSetRAP2(pcg_precond, rap2);
+         HYPRE_BoomerAMGSetKeepTranspose(pcg_precond, keepTranspose);
+         if (nongalerk_tol)
+         {
+            HYPRE_BoomerAMGSetNonGalerkinTol(pcg_precond, nongalerk_tol[nongalerk_num_tol-1]);
+            for (i=0; i < nongalerk_num_tol-1; i++)
+               HYPRE_BoomerAMGSetLevelNonGalerkinTol(pcg_precond, nongalerk_tol[i], i);
+         }
+         HYPRE_BiCGSTABSetMaxIter(pcg_solver, mg_max_iter);
          HYPRE_BiCGSTABSetPrecond(pcg_solver,
                                   (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
                                   (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
@@ -3613,8 +4836,8 @@ main( hypre_int argc,
          pcg_precond = NULL;
 
          HYPRE_BiCGSTABSetPrecond(pcg_solver,
-                          (HYPRE_PtrToSolverFcn) HYPRE_ParCSRDiagScale,
-                          (HYPRE_PtrToSolverFcn) HYPRE_ParCSRDiagScaleSetup,
+                                  (HYPRE_PtrToSolverFcn) HYPRE_ParCSRDiagScale,
+                                  (HYPRE_PtrToSolverFcn) HYPRE_ParCSRDiagScaleSetup,
                                   pcg_precond);
       }
       else if (solver_id == 11)
@@ -3624,7 +4847,7 @@ main( hypre_int argc,
 
          ierr = HYPRE_ParCSRPilutCreate( hypre_MPI_COMM_WORLD, &pcg_precond ); 
          if (ierr) {
-	   hypre_printf("Error in ParPilutCreate\n");
+            hypre_printf("Error in ParPilutCreate\n");
          }
 
          HYPRE_BiCGSTABSetPrecond(pcg_solver,
@@ -3634,11 +4857,11 @@ main( hypre_int argc,
 
          if (drop_tol >= 0 )
             HYPRE_ParCSRPilutSetDropTolerance( pcg_precond,
-               drop_tol );
+                                               drop_tol );
 
          if (nonzeros_to_keep >= 0 )
             HYPRE_ParCSRPilutSetFactorRowSize( pcg_precond,
-               nonzeros_to_keep );
+                                               nonzeros_to_keep );
       }
       else if (solver_id == 45)
       {
@@ -3652,7 +4875,15 @@ main( hypre_int argc,
             we'll use what I think is simplest: let Euclid internally 
             parse the command line.
          */   
-         HYPRE_EuclidSetParams(pcg_precond, argc, argv);
+         if (eu_level > -1) HYPRE_EuclidSetLevel(pcg_precond, eu_level);
+         if (eu_ilut) HYPRE_EuclidSetILUT(pcg_precond, eu_ilut);
+         if (eu_sparse_A) HYPRE_EuclidSetSparseA(pcg_precond, eu_sparse_A);
+         if (eu_row_scale) HYPRE_EuclidSetRowScale(pcg_precond, eu_row_scale);
+         if (eu_bj) HYPRE_EuclidSetBJ(pcg_precond, eu_bj);
+         HYPRE_EuclidSetStats(pcg_precond, eu_stats);
+         HYPRE_EuclidSetMem(pcg_precond, eu_mem);
+
+         /*HYPRE_EuclidSetParams(pcg_precond, argc, argv);*/
 
          HYPRE_BiCGSTABSetPrecond(pcg_solver,
                                   (HYPRE_PtrToSolverFcn) HYPRE_EuclidSolve,
@@ -3661,7 +4892,7 @@ main( hypre_int argc,
       }
  
       HYPRE_BiCGSTABSetup(pcg_solver, (HYPRE_Matrix)parcsr_A, 
-		(HYPRE_Vector)b, (HYPRE_Vector)x);
+                          (HYPRE_Vector)b, (HYPRE_Vector)x);
  
       hypre_EndTiming(time_index);
       hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
@@ -3672,7 +4903,7 @@ main( hypre_int argc,
       hypre_BeginTiming(time_index);
  
       HYPRE_BiCGSTABSolve(pcg_solver, (HYPRE_Matrix)parcsr_A, 
-		(HYPRE_Vector)b, (HYPRE_Vector)x);
+                          (HYPRE_Vector)b, (HYPRE_Vector)x);
  
       hypre_EndTiming(time_index);
       hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
@@ -3685,9 +4916,9 @@ main( hypre_int argc,
       /* run a second time to check for memory leaks */
       HYPRE_ParVectorSetRandomValues(x, 775);
       HYPRE_BiCGSTABSetup(pcg_solver, (HYPRE_Matrix)parcsr_A, 
-		(HYPRE_Vector)b, (HYPRE_Vector)x);
+                          (HYPRE_Vector)b, (HYPRE_Vector)x);
       HYPRE_BiCGSTABSolve(pcg_solver, (HYPRE_Matrix)parcsr_A, 
-		(HYPRE_Vector)b, (HYPRE_Vector)x);
+                          (HYPRE_Vector)b, (HYPRE_Vector)x);
 #endif
 
       HYPRE_ParCSRBiCGSTABDestroy(pcg_solver);
@@ -3703,7 +4934,7 @@ main( hypre_int argc,
       }
       else if (solver_id == 45)
       {
-        HYPRE_EuclidDestroy(pcg_precond);
+         HYPRE_EuclidDestroy(pcg_precond);
       }
 
       if (myid == 0)
@@ -3724,7 +4955,7 @@ main( hypre_int argc,
       hypre_BeginTiming(time_index);
  
       HYPRE_ParCSRCGNRCreate(hypre_MPI_COMM_WORLD, &pcg_solver);
-      HYPRE_CGNRSetMaxIter(pcg_solver, 1000);
+      HYPRE_CGNRSetMaxIter(pcg_solver, max_iter);
       HYPRE_CGNRSetTol(pcg_solver, tol);
       HYPRE_CGNRSetLogging(pcg_solver, ioutdat);
  
@@ -3733,40 +4964,80 @@ main( hypre_int argc,
          /* use BoomerAMG as preconditioner */
          if (myid == 0) hypre_printf("Solver: AMG-CGNR\n");
          HYPRE_BoomerAMGCreate(&pcg_precond); 
-         /* BM Aug 25, 2006 */
          HYPRE_BoomerAMGSetCGCIts(pcg_precond, cgcits);
          HYPRE_BoomerAMGSetInterpType(pcg_precond, interp_type);
+         HYPRE_BoomerAMGSetPostInterpType(pcg_precond, post_interp_type);
          HYPRE_BoomerAMGSetNumSamples(pcg_precond, gsmg_samples);
          HYPRE_BoomerAMGSetTol(pcg_precond, pc_tol);
-         HYPRE_BoomerAMGSetCoarsenType(pcg_precond, (hybrid*coarsen_type));
+         HYPRE_BoomerAMGSetCoarsenType(pcg_precond, coarsen_type);
          HYPRE_BoomerAMGSetMeasureType(pcg_precond, measure_type);
          HYPRE_BoomerAMGSetStrongThreshold(pcg_precond, strong_threshold);
+         HYPRE_BoomerAMGSetSeqThreshold(pcg_precond, seq_threshold);
+         HYPRE_BoomerAMGSetRedundant(pcg_precond, redundant);
+         HYPRE_BoomerAMGSetMaxCoarseSize(pcg_precond, coarse_threshold);
+         HYPRE_BoomerAMGSetMinCoarseSize(pcg_precond, min_coarse_size);
          HYPRE_BoomerAMGSetTruncFactor(pcg_precond, trunc_factor);
          HYPRE_BoomerAMGSetPMaxElmts(pcg_precond, P_max_elmts);
-         HYPRE_BoomerAMGSetPostInterpType(pcg_precond, post_interp_type);
          HYPRE_BoomerAMGSetJacobiTruncThreshold(pcg_precond, jacobi_trunc_threshold);
+         HYPRE_BoomerAMGSetSCommPkgSwitch(pcg_precond, S_commpkg_switch);
          HYPRE_BoomerAMGSetPrintLevel(pcg_precond, poutdat);
          HYPRE_BoomerAMGSetPrintFileName(pcg_precond, "driver.out.log");
          HYPRE_BoomerAMGSetMaxIter(pcg_precond, 1);
          HYPRE_BoomerAMGSetCycleType(pcg_precond, cycle_type);
-         HYPRE_BoomerAMGSetNumGridSweeps(pcg_precond, num_grid_sweeps);
-         HYPRE_BoomerAMGSetGridRelaxType(pcg_precond, grid_relax_type);
-         HYPRE_BoomerAMGSetRelaxWeight(pcg_precond, relax_weight);
-         HYPRE_BoomerAMGSetOmega(pcg_precond, omega);
+         HYPRE_BoomerAMGSetNumSweeps(pcg_precond, num_sweeps);
+         if (relax_type > -1) HYPRE_BoomerAMGSetRelaxType(pcg_precond, relax_type);
+         if (relax_down > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_down, 1);
+         if (relax_up > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_up, 2);
+         if (relax_coarse > -1)
+            HYPRE_BoomerAMGSetCycleRelaxType(pcg_precond, relax_coarse, 3);
+         HYPRE_BoomerAMGSetAddRelaxType(pcg_precond, add_relax_type);
+         HYPRE_BoomerAMGSetAddRelaxWt(pcg_precond, add_relax_wt);
+         HYPRE_BoomerAMGSetChebyOrder(pcg_precond, cheby_order);
+         HYPRE_BoomerAMGSetChebyFraction(pcg_precond, cheby_fraction);
+         HYPRE_BoomerAMGSetRelaxOrder(pcg_precond, relax_order);
+         HYPRE_BoomerAMGSetRelaxWt(pcg_precond, relax_wt);
+         HYPRE_BoomerAMGSetOuterWt(pcg_precond, outer_wt);
+         if (level_w > -1)
+            HYPRE_BoomerAMGSetLevelRelaxWt(pcg_precond, relax_wt_level,level_w);
+         if (level_ow > -1)
+            HYPRE_BoomerAMGSetLevelOuterWt(pcg_precond,outer_wt_level,level_ow);
          HYPRE_BoomerAMGSetSmoothType(pcg_precond, smooth_type);
          HYPRE_BoomerAMGSetSmoothNumLevels(pcg_precond, smooth_num_levels);
          HYPRE_BoomerAMGSetSmoothNumSweeps(pcg_precond, smooth_num_sweeps);
-         HYPRE_BoomerAMGSetGridRelaxPoints(pcg_precond, grid_relax_points);
          HYPRE_BoomerAMGSetMaxLevels(pcg_precond, max_levels);
          HYPRE_BoomerAMGSetMaxRowSum(pcg_precond, max_row_sum);
+         HYPRE_BoomerAMGSetDebugFlag(pcg_precond, debug_flag);
          HYPRE_BoomerAMGSetNumFunctions(pcg_precond, num_functions);
-         HYPRE_BoomerAMGSetNumPaths(pcg_precond, num_paths);
          HYPRE_BoomerAMGSetAggNumLevels(pcg_precond, agg_num_levels);
+         HYPRE_BoomerAMGSetAggInterpType(pcg_precond, agg_interp_type);
+         HYPRE_BoomerAMGSetAggTruncFactor(pcg_precond, agg_trunc_factor);
+         HYPRE_BoomerAMGSetAggP12TruncFactor(pcg_precond, agg_P12_trunc_factor);
+         HYPRE_BoomerAMGSetAggPMaxElmts(pcg_precond, agg_P_max_elmts);
+         HYPRE_BoomerAMGSetAggP12MaxElmts(pcg_precond, agg_P12_max_elmts);
+         HYPRE_BoomerAMGSetNumPaths(pcg_precond, num_paths);
+         HYPRE_BoomerAMGSetNodal(pcg_precond, nodal);
+         HYPRE_BoomerAMGSetNodalDiag(pcg_precond, nodal_diag);
          HYPRE_BoomerAMGSetVariant(pcg_precond, variant);
          HYPRE_BoomerAMGSetOverlap(pcg_precond, overlap);
          HYPRE_BoomerAMGSetDomainType(pcg_precond, domain_type);
          if (num_functions > 1)
             HYPRE_BoomerAMGSetDofFunc(pcg_precond, dof_func);
+         HYPRE_BoomerAMGSetAdditive(pcg_precond, additive);
+         HYPRE_BoomerAMGSetMultAdditive(pcg_precond, mult_add);
+         HYPRE_BoomerAMGSetSimple(pcg_precond, simple);
+         HYPRE_BoomerAMGSetMultAddPMaxElmts(pcg_precond, add_P_max_elmts);
+         HYPRE_BoomerAMGSetMultAddTruncFactor(pcg_precond, add_trunc_factor);
+         HYPRE_BoomerAMGSetRAP2(pcg_precond, rap2);
+         HYPRE_BoomerAMGSetKeepTranspose(pcg_precond, keepTranspose);
+         if (nongalerk_tol)
+         {
+            HYPRE_BoomerAMGSetNonGalerkinTol(pcg_precond, nongalerk_tol[nongalerk_num_tol-1]);
+            for (i=0; i < nongalerk_num_tol-1; i++)
+               HYPRE_BoomerAMGSetLevelNonGalerkinTol(pcg_precond, nongalerk_tol[i], i);
+         }
+         HYPRE_CGNRSetMaxIter(pcg_solver, mg_max_iter);
          HYPRE_CGNRSetPrecond(pcg_solver,
                               (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
                               (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolveT,
@@ -3789,14 +5060,14 @@ main( hypre_int argc,
       HYPRE_CGNRGetPrecond(pcg_solver, &pcg_precond_gotten);
       if (pcg_precond_gotten != pcg_precond)
       {
-        hypre_printf("HYPRE_ParCSRCGNRGetPrecond got bad precond\n");
-        return(-1);
+         hypre_printf("HYPRE_ParCSRCGNRGetPrecond got bad precond\n");
+         return(-1);
       }
       else
-        if (myid == 0)
-          hypre_printf("HYPRE_ParCSRCGNRGetPrecond got good precond\n");
+         if (myid == 0)
+            hypre_printf("HYPRE_ParCSRCGNRGetPrecond got good precond\n");
       HYPRE_CGNRSetup(pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, 
-		(HYPRE_Vector)x);
+                      (HYPRE_Vector)x);
  
       hypre_EndTiming(time_index);
       hypre_PrintTiming("Setup phase times", hypre_MPI_COMM_WORLD);
@@ -3807,7 +5078,7 @@ main( hypre_int argc,
       hypre_BeginTiming(time_index);
  
       HYPRE_CGNRSolve(pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, 
-		(HYPRE_Vector)x);
+                      (HYPRE_Vector)x);
  
       hypre_EndTiming(time_index);
       hypre_PrintTiming("Solve phase times", hypre_MPI_COMM_WORLD);
@@ -3821,9 +5092,9 @@ main( hypre_int argc,
       /* run a second time to check for memory leaks */
       HYPRE_ParVectorSetRandomValues(x, 775);
       HYPRE_CGNRSetup(pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, 
-		(HYPRE_Vector)x);
+                      (HYPRE_Vector)x);
       HYPRE_CGNRSolve(pcg_solver, (HYPRE_Matrix)parcsr_A, (HYPRE_Vector)b, 
-		(HYPRE_Vector)x);
+                      (HYPRE_Vector)x);
 #endif
 
       HYPRE_ParCSRCGNRDestroy(pcg_solver);
@@ -3845,20 +5116,41 @@ main( hypre_int argc,
     * Print the solution and other info
     *-----------------------------------------------------------*/
 
-   HYPRE_IJVectorGetObjectType(ij_b, &j);
-   /* HYPRE_IJVectorPrint(ij_b, "driver.out.b");
-   HYPRE_IJVectorPrint(ij_x, "driver.out.x"); */
+   /* RDF: Why is this here? */
+   if (!(build_rhs_type ==1 || build_rhs_type ==7))
+      HYPRE_IJVectorGetObjectType(ij_b, &j);
+
+   if (print_system)
+   {
+      HYPRE_IJVectorPrint(ij_x, "IJ.out.x");
+   }
 
    /*-----------------------------------------------------------
     * Finalize things
     *-----------------------------------------------------------*/
 
-   HYPRE_IJMatrixDestroy(ij_A);
-   HYPRE_IJVectorDestroy(ij_b);
+   if (test_ij || build_matrix_type == -1) HYPRE_IJMatrixDestroy(ij_A);
+   else HYPRE_ParCSRMatrixDestroy(parcsr_A);
+
+   /* for build_rhs_type = 1 or 7, we did not create ij_b  - just b*/
+   if (build_rhs_type ==1 || build_rhs_type ==7)
+      HYPRE_ParVectorDestroy(b);
+   else
+      HYPRE_IJVectorDestroy(ij_b);
+
    HYPRE_IJVectorDestroy(ij_x);
 
+   if (build_rbm) 
+   {
+      for (i=0; i< num_interp_vecs; i++) 
+         HYPRE_IJVectorDestroy(ij_rbm[i]);
+      hypre_TFree(ij_rbm);
+      hypre_TFree(interp_vecs);
+   }
+   if (nongalerk_tol) hypre_TFree (nongalerk_tol);
+
 /*
-   hypre_FinalizeMemoryDebug();
+  hypre_FinalizeMemoryDebug();
 */
 
    hypre_MPI_Finalize();
@@ -3927,58 +5219,130 @@ BuildParFromFile( HYPRE_Int                  argc,
    return (0);
 }
 
+
 /*----------------------------------------------------------------------
- * Build standard 7-point laplacian in 3D with grid and anisotropy.
- * Parameters given in command line.
+ * Build rhs from file. Expects two files on each processor.
+ * filename.n contains the data and
+ * and filename.INFO.n contains global row
+ * numbers
  *----------------------------------------------------------------------*/
 
 HYPRE_Int
-BuildParLaplacian( HYPRE_Int                  argc,
-                   char                *argv[],
-                   HYPRE_Int                  arg_index,
-                   HYPRE_ParCSRMatrix  *A_ptr     )
+BuildParRhsFromFile( HYPRE_Int                  argc,
+                     char                *argv[],
+                     HYPRE_Int                  arg_index,
+                     HYPRE_ParVector      *b_ptr     )
 {
-   HYPRE_Int                 nx, ny, nz;
-   HYPRE_Int                 P, Q, R;
-   HYPRE_Real          cx, cy, cz;
+   char               *filename;
 
-   HYPRE_ParCSRMatrix  A;
+   HYPRE_ParVector b;
 
-   HYPRE_Int                 num_procs, myid;
-   HYPRE_Int                 p, q, r;
-   HYPRE_Real         *values;
+   HYPRE_Int                 myid;
 
    /*-----------------------------------------------------------
     * Initialize some stuff
     *-----------------------------------------------------------*/
 
-   hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
    hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &myid );
 
    /*-----------------------------------------------------------
-    * Set defaults
+    * Parse command line
     *-----------------------------------------------------------*/
- 
-   nx = 10;
-   ny = 10;
-   nz = 10;
 
-   P  = 1;
-   Q  = num_procs;
-   R  = 1;
-
-   cx = 1.;
-   cy = 1.;
-   cz = 1.;
+   if (arg_index < argc)
+   {
+      filename = argv[arg_index];
+   }
+   else
+   {
+      hypre_printf("Error: No filename specified \n");
+      exit(1);
+   }
 
    /*-----------------------------------------------------------
-    * Parse command line
+    * Print driver parameters
     *-----------------------------------------------------------*/
-   arg_index = 0;
-   while (arg_index < argc)
+ 
+   if (myid == 0)
    {
-      if ( strcmp(argv[arg_index], "-n") == 0 )
-      {
+      hypre_printf("  RhsFromParFile: %s\n", filename);
+   }
+
+   /*-----------------------------------------------------------
+    * Generate the matrix 
+    *-----------------------------------------------------------*/
+ 
+   HYPRE_ParVectorRead(hypre_MPI_COMM_WORLD, filename,&b);
+
+   *b_ptr = b;
+
+   return (0);
+}
+
+
+
+
+/*----------------------------------------------------------------------
+ * Build standard 7-point laplacian in 3D with grid and anisotropy.
+ * Parameters given in command line.
+ *----------------------------------------------------------------------*/
+
+HYPRE_Int
+BuildParLaplacian( HYPRE_Int                  argc,
+                   char                *argv[],
+                   HYPRE_Int                  arg_index,
+                   HYPRE_ParCSRMatrix  *A_ptr     )
+{
+   HYPRE_Int                 nx, ny, nz;
+   HYPRE_Int                 P, Q, R;
+   HYPRE_Real          cx, cy, cz;
+
+   HYPRE_ParCSRMatrix  A;
+
+   HYPRE_Int                 num_procs, myid;
+   HYPRE_Int                 p, q, r;
+   HYPRE_Int                 num_fun = 1;
+   HYPRE_Real         *values;
+   HYPRE_Real         *mtrx;
+
+   HYPRE_Real          ep = .1;
+   
+   HYPRE_Int                 system_vcoef = 0;
+   HYPRE_Int                 sys_opt = 0;
+   HYPRE_Int                 vcoef_opt = 0;
+   
+   
+   /*-----------------------------------------------------------
+    * Initialize some stuff
+    *-----------------------------------------------------------*/
+
+   hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
+   hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &myid );
+
+   /*-----------------------------------------------------------
+    * Set defaults
+    *-----------------------------------------------------------*/
+ 
+   nx = 10;
+   ny = 10;
+   nz = 10;
+
+   P  = 1;
+   Q  = num_procs;
+   R  = 1;
+
+   cx = 1.;
+   cy = 1.;
+   cz = 1.;
+
+   /*-----------------------------------------------------------
+    * Parse command line
+    *-----------------------------------------------------------*/
+   arg_index = 0;
+   while (arg_index < argc)
+   {
+      if ( strcmp(argv[arg_index], "-n") == 0 )
+      {
          arg_index++;
          nx = atoi(argv[arg_index++]);
          ny = atoi(argv[arg_index++]);
@@ -3998,6 +5362,32 @@ BuildParLaplacian( HYPRE_Int                  argc,
          cy = atof(argv[arg_index++]);
          cz = atof(argv[arg_index++]);
       }
+      else if ( strcmp(argv[arg_index], "-sysL") == 0 )
+      {
+         arg_index++;
+         num_fun = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-sysL_opt") == 0 )
+      {
+         arg_index++;
+         sys_opt = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-sys_vcoef") == 0 )
+      {
+         /* have to use -sysL for this to */
+         arg_index++;
+         system_vcoef = 1;
+      }
+      else if ( strcmp(argv[arg_index], "-sys_vcoef_opt") == 0 )
+      {
+         arg_index++;
+         vcoef_opt = atoi(argv[arg_index++]); 
+      }
+      else if ( strcmp(argv[arg_index], "-ep") == 0 )
+      {
+         arg_index++;
+         ep = atof(argv[arg_index++]); 
+      }
       else
       {
          arg_index++;
@@ -4020,7 +5410,7 @@ BuildParLaplacian( HYPRE_Int                  argc,
  
    if (myid == 0)
    {
-      hypre_printf("  Laplacian:\n");
+      hypre_printf("  Laplacian:   num_fun = %d\n", num_fun);
       hypre_printf("    (nx, ny, nz) = (%d, %d, %d)\n", nx, ny, nz);
       hypre_printf("    (Px, Py, Pz) = (%d, %d, %d)\n", P,  Q,  R);
       hypre_printf("    (cx, cy, cz) = (%f, %f, %f)\n\n", cx, cy, cz);
@@ -4059,8 +5449,310 @@ BuildParLaplacian( HYPRE_Int                  argc,
       values[0] += 2.0*cz;
    }
 
-   A = (HYPRE_ParCSRMatrix) GenerateLaplacian(hypre_MPI_COMM_WORLD, 
-		nx, ny, nz, P, Q, R, p, q, r, values);
+   if (num_fun == 1)
+      A = (HYPRE_ParCSRMatrix) GenerateLaplacian(hypre_MPI_COMM_WORLD, 
+                                                 nx, ny, nz, P, Q, R, p, q, r, values);
+   else
+   {
+      mtrx = hypre_CTAlloc(HYPRE_Real, num_fun*num_fun);
+
+      if (num_fun == 2)
+      {
+         if (sys_opt ==1) /* identity  */
+         {
+            mtrx[0] = 1.0;
+            mtrx[1] = 0.0; 
+            mtrx[2] = 0.0;
+            mtrx[3] = 1.0; 
+         }
+         else if (sys_opt ==2)
+         {
+            mtrx[0] = 1.0;
+            mtrx[1] = 0.0; 
+            mtrx[2] = 0.0;
+            mtrx[3] = 20.0; 
+         }
+         else if (sys_opt ==3) /* similar to barry's talk - ex1 */
+         {
+            mtrx[0] = 1.0;
+            mtrx[1] = 2.0; 
+            mtrx[2] = 2.0;
+            mtrx[3] = 1.0; 
+         }
+         else if (sys_opt ==4) /* can use with vcoef to get barry's ex*/
+         {
+            mtrx[0] = 1.0;
+            mtrx[1] = 1.0; 
+            mtrx[2] = 1.0;
+            mtrx[3] = 1.0; 
+         }
+         else if (sys_opt ==5) /* barry's talk - ex1 */
+         {
+            mtrx[0] = 1.0;
+            mtrx[1] = 1.1; 
+            mtrx[2] = 1.1;
+            mtrx[3] = 1.0; 
+         }
+         else if (sys_opt ==6) /*  */
+         {
+            mtrx[0] = 1.1;
+            mtrx[1] = 1.0; 
+            mtrx[2] = 1.0;
+            mtrx[3] = 1.1; 
+         }
+
+         else /* == 0 */
+         {
+            mtrx[0] = 2;
+            mtrx[1] = 1;
+            mtrx[2] = 1;
+            mtrx[3] = 2;
+         }
+      }
+      else if (num_fun == 3)
+      {
+         if (sys_opt ==1)
+         {
+            mtrx[0] = 1.0;
+            mtrx[1] = 0.0;
+            mtrx[2] = 0.0;
+            mtrx[3] = 0.0;
+            mtrx[4] = 1.0;
+            mtrx[5] = 0.0;
+            mtrx[6] = 0.0;
+            mtrx[7] = 0.0;
+            mtrx[8] = 1.0;
+         }
+         else if (sys_opt ==2)
+         {
+            mtrx[0] = 1.0;
+            mtrx[1] = 0.0;
+            mtrx[2] = 0.0;
+            mtrx[3] = 0.0;
+            mtrx[4] = 20.0;
+            mtrx[5] = 0.0;
+            mtrx[6] = 0.0;
+            mtrx[7] = 0.0;
+            mtrx[8] =.01;
+         }
+         else if (sys_opt ==3)
+         {
+            mtrx[0] = 1.01;
+            mtrx[1] = 1;
+            mtrx[2] = 0.0;
+            mtrx[3] = 1;
+            mtrx[4] = 2;
+            mtrx[5] = 1;
+            mtrx[6] = 0.0;
+            mtrx[7] = 1;
+            mtrx[8] = 1.01;  
+         }
+         else if (sys_opt ==4) /* barry ex4 */
+         {
+            mtrx[0] = 3;
+            mtrx[1] = 1;
+            mtrx[2] = 0.0;
+            mtrx[3] = 1;
+            mtrx[4] = 4;
+            mtrx[5] = 2;
+            mtrx[6] = 0.0;
+            mtrx[7] = 2;
+            mtrx[8] = .25;  
+         }
+         else /* == 0 */
+         {
+            mtrx[0] = 2.0;
+            mtrx[1] = 1.0;
+            mtrx[2] = 0.0;
+            mtrx[3] = 1.0;
+            mtrx[4] = 2.0;
+            mtrx[5] = 1.0;
+            mtrx[6] = 0.0;
+            mtrx[7] = 1.0;
+            mtrx[8] = 2.0;
+         }
+
+      } 
+      else if (num_fun == 4)
+      {
+         mtrx[0] = 1.01;
+         mtrx[1] = 1;
+         mtrx[2] = 0.0;
+         mtrx[3] = 0.0;
+         mtrx[4] = 1;
+         mtrx[5] = 2;
+         mtrx[6] = 1;
+         mtrx[7] = 0.0;
+         mtrx[8] = 0.0;
+         mtrx[9] = 1;
+         mtrx[10] = 1.01;
+         mtrx[11] = 0.0;
+         mtrx[12] = 2;
+         mtrx[13] = 1;
+         mtrx[14] = 0.0;
+         mtrx[15] = 1;
+      } 
+
+
+
+
+      if (!system_vcoef)
+      {
+         A = (HYPRE_ParCSRMatrix) GenerateSysLaplacian(hypre_MPI_COMM_WORLD, 
+                                                       nx, ny, nz, P, Q, 
+                                                       R, p, q, r, num_fun, mtrx, values);
+      }
+      else
+      {
+       
+    
+         HYPRE_Real *mtrx_values;
+
+         mtrx_values = hypre_CTAlloc(HYPRE_Real, num_fun*num_fun*4);
+
+         if (num_fun == 2)
+         {
+            if (vcoef_opt == 1)
+            {
+               /* Barry's talk * - must also have sys_opt = 4, all fail */
+               mtrx[0] = 1.0;
+               SetSysVcoefValues(num_fun, nx, ny, nz, 1.0, .10, 1.0, 0, mtrx_values);
+               
+               mtrx[1]  = 1.0;
+               SetSysVcoefValues(num_fun, nx, ny, nz, .1, 1.0, 1.0, 1, mtrx_values);
+               
+               mtrx[2] = 1.0;
+               SetSysVcoefValues(num_fun, nx, ny, nz, .01, 1.0, 1.0, 2, mtrx_values);
+               
+               mtrx[3] = 1.0;
+               SetSysVcoefValues(num_fun, nx, ny, nz, 2.0, .02, 1.0, 3, mtrx_values);
+               
+            }
+            else if (vcoef_opt == 2)
+            {
+               /* Barry's talk * - ex2 - if have sys-opt = 4*/
+               mtrx[0] = 1.0;
+               SetSysVcoefValues(num_fun, nx, ny, nz, 1.0, .010, 1.0, 0, mtrx_values);
+               
+               mtrx[1]  = 200.0;
+               SetSysVcoefValues(num_fun, nx, ny, nz, 1.0, 1.0, 1.0, 1, mtrx_values);
+
+               mtrx[2] = 200.0;
+               SetSysVcoefValues(num_fun, nx, ny, nz, 1.0, 1.0, 1.0, 2, mtrx_values);
+               
+               mtrx[3] = 1.0;
+               SetSysVcoefValues(num_fun, nx, ny, nz, 2.0, .02, 1.0, 3, mtrx_values);
+               
+            }
+            else if (vcoef_opt == 3) /* use with default sys_opt  - ulrike ex 3*/
+            {
+               
+               /* mtrx[0] */
+               SetSysVcoefValues(num_fun, nx, ny, nz, ep*1.0, 1.0, 1.0, 0, mtrx_values);
+               
+               /* mtrx[1] */
+               SetSysVcoefValues(num_fun, nx, ny, nz, 1.0, 1.0, 1.0, 1, mtrx_values);
+               
+               /* mtrx[2] */
+               SetSysVcoefValues(num_fun, nx, ny, nz, ep*1.0, 1.0, 1.0, 2, mtrx_values);
+               
+               /* mtrx[3] */
+               SetSysVcoefValues(num_fun, nx, ny, nz, 1.0, 1.0, 1.0, 3, mtrx_values);
+            }
+            else if (vcoef_opt == 4) /* use with default sys_opt  - ulrike ex 4*/
+            {
+               HYPRE_Real ep2 = ep;
+               
+               /* mtrx[0] */
+               SetSysVcoefValues(num_fun, nx, ny, nz, ep*1.0, 1.0, 1.0, 0, mtrx_values);
+               
+               /* mtrx[1] */
+               SetSysVcoefValues(num_fun, nx, ny, nz, 1.0, ep*1.0, 1.0, 1, mtrx_values);
+               
+               /* mtrx[2] */
+               SetSysVcoefValues(num_fun, nx, ny, nz, ep*1.0, 1.0, 1.0, 2, mtrx_values);
+               
+               /* mtrx[3] */
+               SetSysVcoefValues(num_fun, nx, ny, nz, 1.0, ep2*1.0, 1.0, 3, mtrx_values);
+            }
+            else if (vcoef_opt == 5) /* use with default sys_opt  - */
+            {
+               HYPRE_Real  alp, beta;
+               alp = .001;
+               beta = 10;
+               
+               /* mtrx[0] */
+               SetSysVcoefValues(num_fun, nx, ny, nz, alp*1.0, 1.0, 1.0, 0, mtrx_values);
+               
+               /* mtrx[1] */
+               SetSysVcoefValues(num_fun, nx, ny, nz, 1.0, beta*1.0, 1.0, 1, mtrx_values);
+               
+               /* mtrx[2] */
+               SetSysVcoefValues(num_fun, nx, ny, nz, alp*1.0, 1.0, 1.0, 2, mtrx_values);
+               
+               /* mtrx[3] */
+               SetSysVcoefValues(num_fun, nx, ny, nz, 1.0, beta*1.0, 1.0, 3, mtrx_values);
+            }
+            else  /* = 0 */
+            {
+               /* mtrx[0] */
+               SetSysVcoefValues(num_fun, nx, ny, nz, 1.0, 1.0, 1.0, 0, mtrx_values);
+               
+               /* mtrx[1] */
+               SetSysVcoefValues(num_fun, nx, ny, nz, 1.0, 2.0, 1.0, 1, mtrx_values);
+               
+               /* mtrx[2] */
+               SetSysVcoefValues(num_fun, nx, ny, nz, 2.0, 1.0, 0.0, 2, mtrx_values);
+               
+               /* mtrx[3] */
+               SetSysVcoefValues(num_fun, nx, ny, nz, 1.0, 3.0, 1.0, 3, mtrx_values);
+            }
+
+         }
+         else if (num_fun == 3)
+         {
+            mtrx[0] = 1;
+            SetSysVcoefValues(num_fun, nx, ny, nz, 1, .01, 1, 0, mtrx_values);
+
+            mtrx[1] = 1;
+            SetSysVcoefValues(num_fun, nx, ny, nz, 1, 1, 1, 1, mtrx_values);
+
+            mtrx[2] = 0.0;
+            SetSysVcoefValues(num_fun, nx, ny, nz, 1, 1, 1, 2, mtrx_values);
+
+            mtrx[3] = 1;
+            SetSysVcoefValues(num_fun, nx, ny, nz, 1, 1, 1, 3, mtrx_values);
+
+            mtrx[4] = 1;
+            SetSysVcoefValues(num_fun, nx, ny, nz, 2, .02, 1, 4, mtrx_values);
+
+            mtrx[5] = 2;
+            SetSysVcoefValues(num_fun, nx, ny, nz, 1, 1, 1, 5, mtrx_values);
+
+            mtrx[6] = 0.0;
+            SetSysVcoefValues(num_fun, nx, ny, nz, 1, 1, 1, 6, mtrx_values);
+
+            mtrx[7] = 2;
+            SetSysVcoefValues(num_fun, nx, ny, nz, 1, 1, 1, 7, mtrx_values);
+
+            mtrx[8] = 1;
+            SetSysVcoefValues(num_fun, nx, ny, nz, 1.5, .04, 1, 8, mtrx_values);
+
+         }
+
+         A = (HYPRE_ParCSRMatrix) GenerateSysLaplacianVCoef(hypre_MPI_COMM_WORLD, 
+                                                            nx, ny, nz, P, Q, 
+                                                            R, p, q, r, num_fun, mtrx, mtrx_values);
+
+
+
+
+
+         hypre_TFree(mtrx_values);
+      }
+
+      hypre_TFree(mtrx);
+   }
 
    hypre_TFree(values);
 
@@ -4228,7 +5920,7 @@ BuildParDifConv( HYPRE_Int                  argc,
    }
 
    A = (HYPRE_ParCSRMatrix) GenerateDifConv(hypre_MPI_COMM_WORLD,
-                               nx, ny, nz, P, Q, R, p, q, r, values);
+                                            nx, ny, nz, P, Q, R, p, q, r, values);
 
    hypre_TFree(values);
 
@@ -4245,7 +5937,7 @@ BuildParDifConv( HYPRE_Int                  argc,
  *----------------------------------------------------------------------*/
 
 HYPRE_Int
-BuildParFromOneFile2(HYPRE_Int                  argc,
+BuildParFromOneFile( HYPRE_Int                  argc,
                      char                *argv[],
                      HYPRE_Int                  arg_index,
                      HYPRE_Int                  num_functions,
@@ -4347,8 +6039,8 @@ BuildFuncsFromFiles(    HYPRE_Int                  argc,
  * Build Function array from files on different processors
  *----------------------------------------------------------------------*/
 
-	hypre_printf (" Feature is not implemented yet!\n");	
-	return(0);
+   hypre_printf (" Feature is not implemented yet!\n");	
+   return(0);
 
 }
 
@@ -4425,22 +6117,22 @@ BuildFuncsFromOneFile(  HYPRE_Int                  argc,
 
    if (myid == 0)
    {
-        requests = hypre_CTAlloc(hypre_MPI_Request,num_procs-1);
-        status = hypre_CTAlloc(hypre_MPI_Status,num_procs-1);
-        j = 0;
-        for (i=1; i < num_procs; i++)
-                hypre_MPI_Isend(&dof_func[partitioning[i]],
-		partitioning[i+1]-partitioning[i],
-                HYPRE_MPI_INT, i, 0, comm, &requests[j++]);
-        for (i=0; i < local_size; i++)
-                dof_func_local[i] = dof_func[i];
-        hypre_MPI_Waitall(num_procs-1,requests, status);
-        hypre_TFree(requests);
-        hypre_TFree(status);
+      requests = hypre_CTAlloc(hypre_MPI_Request,num_procs-1);
+      status = hypre_CTAlloc(hypre_MPI_Status,num_procs-1);
+      j = 0;
+      for (i=1; i < num_procs; i++)
+         hypre_MPI_Isend(&dof_func[partitioning[i]],
+                         partitioning[i+1]-partitioning[i],
+                         HYPRE_MPI_INT, i, 0, comm, &requests[j++]);
+      for (i=0; i < local_size; i++)
+         dof_func_local[i] = dof_func[i];
+      hypre_MPI_Waitall(num_procs-1,requests, status);
+      hypre_TFree(requests);
+      hypre_TFree(status);
    }
    else
    {
-        hypre_MPI_Recv(dof_func_local,local_size,HYPRE_MPI_INT,0,0,comm,&status0);
+      hypre_MPI_Recv(dof_func_local,local_size,HYPRE_MPI_INT,0,0,comm,&status0);
    }
 
    *dof_func_ptr = dof_func_local;
@@ -4456,7 +6148,7 @@ BuildFuncsFromOneFile(  HYPRE_Int                  argc,
  *----------------------------------------------------------------------*/
 
 HYPRE_Int
-BuildRhsParFromOneFile2(HYPRE_Int                  argc,
+BuildRhsParFromOneFile( HYPRE_Int                  argc,
                         char                *argv[],
                         HYPRE_Int                  arg_index,
                         HYPRE_Int                 *partitioning,
@@ -4625,7 +6317,7 @@ BuildParLaplacian9pt( HYPRE_Int                  argc,
    }
 
    A = (HYPRE_ParCSRMatrix) GenerateLaplacian9pt(hypre_MPI_COMM_WORLD,
-                                  nx, ny, P, Q, p, q, values);
+                                                 nx, ny, P, Q, p, q, values);
 
    hypre_TFree(values);
 
@@ -4736,13 +6428,13 @@ BuildParLaplacian27pt( HYPRE_Int                  argc,
 
    values[0] = 26.0;
    if (nx == 1 || ny == 1 || nz == 1)
-	values[0] = 8.0;
+      values[0] = 8.0;
    if (nx*ny == 1 || nx*nz == 1 || ny*nz == 1)
-	values[0] = 2.0;
+      values[0] = 2.0;
    values[1] = -1.;
 
    A = (HYPRE_ParCSRMatrix) GenerateLaplacian27pt(hypre_MPI_COMM_WORLD,
-                               nx, ny, nz, P, Q, R, p, q, r, values);
+                                                  nx, ny, nz, P, Q, R, p, q, r, values);
 
    hypre_TFree(values);
 
@@ -4751,13 +6443,369 @@ BuildParLaplacian27pt( HYPRE_Int                  argc,
    return (0);
 }
 
-/* begin lobpcg */
 
 /*----------------------------------------------------------------------
- * Build standard 7-point laplacian in 3D.
+ * Build 7-point in 2D 
+ * Parameters given in command line.
  *----------------------------------------------------------------------*/
 
 HYPRE_Int
+BuildParRotate7pt( HYPRE_Int                  argc,
+                   char                *argv[],
+                   HYPRE_Int                  arg_index,
+                   HYPRE_ParCSRMatrix  *A_ptr     )
+{
+   HYPRE_Int                 nx, ny;
+   HYPRE_Int                 P, Q;
+
+   HYPRE_ParCSRMatrix  A;
+
+   HYPRE_Int                 num_procs, myid;
+   HYPRE_Int                 p, q;
+   HYPRE_Real          eps, alpha;
+
+   /*-----------------------------------------------------------
+    * Initialize some stuff
+    *-----------------------------------------------------------*/
+
+   hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
+   hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &myid );
+
+   /*-----------------------------------------------------------
+    * Set defaults
+    *-----------------------------------------------------------*/
+
+   nx = 10;
+   ny = 10;
+
+   P  = 1;
+   Q  = num_procs;
+
+   /*-----------------------------------------------------------
+    * Parse command line
+    *-----------------------------------------------------------*/
+   arg_index = 0;
+   while (arg_index < argc)
+   {
+      if ( strcmp(argv[arg_index], "-n") == 0 )
+      {
+         arg_index++;
+         nx = atoi(argv[arg_index++]);
+         ny = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-P") == 0 )
+      {
+         arg_index++;
+         P  = atoi(argv[arg_index++]);
+         Q  = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-alpha") == 0 )
+      {
+         arg_index++;
+         alpha  = atof(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-eps") == 0 )
+      {
+         arg_index++;
+         eps  = atof(argv[arg_index++]);
+      }
+      else
+      {
+         arg_index++;
+      }
+   }
+
+   /*-----------------------------------------------------------
+    * Check a few things
+    *-----------------------------------------------------------*/
+
+   if ((P*Q) != num_procs)
+   {
+      hypre_printf("Error: Invalid number of processors or processor topology \n");
+      exit(1);
+   }
+
+   /*-----------------------------------------------------------
+    * Print driver parameters
+    *-----------------------------------------------------------*/
+
+   if (myid == 0)
+   {
+      hypre_printf("  Rotate 7pt:\n");
+      hypre_printf("    alpha = %f, eps = %f\n", alpha,eps);
+      hypre_printf("    (nx, ny) = (%d, %d)\n", nx, ny);
+      hypre_printf("    (Px, Py) = (%d, %d)\n", P,  Q);
+   }
+
+   /*-----------------------------------------------------------
+    * Set up the grid structure
+    *-----------------------------------------------------------*/
+
+   /* compute p,q from P,Q and myid */
+   p = myid % P;
+   q = ( myid - p)/P;
+
+   /*-----------------------------------------------------------
+    * Generate the matrix 
+    *-----------------------------------------------------------*/
+
+   A = (HYPRE_ParCSRMatrix) GenerateRotate7pt(hypre_MPI_COMM_WORLD,
+                                              nx, ny, P, Q, p, q, alpha, eps);
+
+   *A_ptr = A;
+
+   return (0);
+}
+
+/*----------------------------------------------------------------------
+ * Build standard 7-point difference operator using centered differences
+ *
+ *  eps*(a(x,y,z) ux)x + (b(x,y,z) uy)y + (c(x,y,z) uz)z 
+ *  d(x,y,z) ux + e(x,y,z) uy + f(x,y,z) uz + g(x,y,z) u
+ *
+ *  functions a,b,c,d,e,f,g need to be defined inside par_vardifconv.c
+ *
+ *----------------------------------------------------------------------*/
+
+HYPRE_Int
+BuildParVarDifConv( HYPRE_Int                  argc,
+                    char                *argv[],
+                    HYPRE_Int                  arg_index,
+                    HYPRE_ParCSRMatrix  *A_ptr    ,
+                    HYPRE_ParVector  *rhs_ptr     )
+{
+   HYPRE_Int                 nx, ny, nz;
+   HYPRE_Int                 P, Q, R;
+
+   HYPRE_ParCSRMatrix  A;
+   HYPRE_ParVector  rhs;
+
+   HYPRE_Int                 num_procs, myid;
+   HYPRE_Int                 p, q, r;
+   HYPRE_Real          eps;
+
+   /*-----------------------------------------------------------
+    * Initialize some stuff
+    *-----------------------------------------------------------*/
+
+   hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
+   hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &myid );
+
+   /*-----------------------------------------------------------
+    * Set defaults
+    *-----------------------------------------------------------*/
+
+   nx = 10;
+   ny = 10;
+   nz = 10;
+   P  = 1;
+   Q  = num_procs;
+   R  = 1;
+
+   /*-----------------------------------------------------------
+    * Parse command line
+    *-----------------------------------------------------------*/
+   arg_index = 0;
+   while (arg_index < argc)
+   {
+      if ( strcmp(argv[arg_index], "-n") == 0 )
+      {
+         arg_index++;
+         nx = atoi(argv[arg_index++]);
+         ny = atoi(argv[arg_index++]);
+         nz = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-P") == 0 )
+      {
+         arg_index++;
+         P  = atoi(argv[arg_index++]);
+         Q  = atoi(argv[arg_index++]);
+         R  = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-eps") == 0 )
+      {
+         arg_index++;
+         eps  = atof(argv[arg_index++]);
+      }
+      else
+      {
+         arg_index++;
+      }
+   }
+
+   /*-----------------------------------------------------------
+    * Check a few things
+    *-----------------------------------------------------------*/
+
+   if ((P*Q*R) != num_procs)
+   {
+      hypre_printf("Error: Invalid number of processors or processor topology \n");
+      exit(1);
+   }
+
+   /*-----------------------------------------------------------
+    * Print driver parameters
+    *-----------------------------------------------------------*/
+
+   if (myid == 0)
+   {
+      hypre_printf("  ell PDE: eps = %f\n", eps);
+      hypre_printf("    Dx(aDxu) + Dy(bDyu) + Dz(cDzu) + d Dxu + e Dyu + f Dzu  + g u= f\n");
+      hypre_printf("    (nx, ny, nz) = (%d, %d, %d)\n", nx, ny, nz);
+      hypre_printf("    (Px, Py, Pz) = (%d, %d, %d)\n", P,  Q,  R);
+   }
+   /*-----------------------------------------------------------
+    * Set up the grid structure
+    *-----------------------------------------------------------*/
+
+   /* compute p,q,r from P,Q,R and myid */
+   p = myid % P;
+   q = (( myid - p)/P) % Q;
+   r = ( myid - p - P*q)/( P*Q );
+
+   /*-----------------------------------------------------------
+    * Generate the matrix
+    *-----------------------------------------------------------*/
+
+   A = (HYPRE_ParCSRMatrix) GenerateVarDifConv(hypre_MPI_COMM_WORLD,
+                                               nx, ny, nz, P, Q, R, p, q, r, eps, &rhs);
+
+   *A_ptr = A;
+   *rhs_ptr = rhs;
+
+   return (0);
+}
+
+/**************************************************************************/
+
+
+HYPRE_Int SetSysVcoefValues(HYPRE_Int num_fun, HYPRE_Int nx, HYPRE_Int ny, HYPRE_Int nz, HYPRE_Real vcx, 
+                            HYPRE_Real vcy, HYPRE_Real vcz, HYPRE_Int mtx_entry, HYPRE_Real *values)
+{
+
+
+   HYPRE_Int sz = num_fun*num_fun;
+
+   values[1*sz + mtx_entry] = -vcx;
+   values[2*sz + mtx_entry] = -vcy;
+   values[3*sz + mtx_entry] = -vcz;
+   values[0*sz + mtx_entry] = 0.0;
+
+   if (nx > 1)
+   {
+      values[0*sz + mtx_entry] += 2.0*vcx;
+   }
+   if (ny > 1)
+   {
+      values[0*sz + mtx_entry] += 2.0*vcy;
+   }
+   if (nz > 1)
+   {
+      values[0*sz + mtx_entry] += 2.0*vcz;
+   }
+
+   return 0;
+   
+}
+                                                                                
+/*----------------------------------------------------------------------
+ * Build coordinates for 1D/2D/3D
+ *----------------------------------------------------------------------*/
+                                                                                
+HYPRE_Int
+BuildParCoordinates( HYPRE_Int                  argc,
+                     char                *argv[],
+                     HYPRE_Int                  arg_index,
+                     HYPRE_Int                 *coorddim_ptr,
+                     float               **coord_ptr     )
+{
+   HYPRE_Int                 nx, ny, nz;
+   HYPRE_Int                 P, Q, R;
+                                                                                
+   HYPRE_Int                 num_procs, myid;
+   HYPRE_Int                 p, q, r;
+                                                                                
+   HYPRE_Int                 coorddim;
+   float               *coordinates;
+                                                                                
+   /*-----------------------------------------------------------
+    * Initialize some stuff
+    *-----------------------------------------------------------*/
+                                                                                
+   hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
+   hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &myid );
+                                                                                
+   /*-----------------------------------------------------------
+    * Set defaults
+    *-----------------------------------------------------------*/
+                                                                                
+   nx = 10;
+   ny = 10;
+   nz = 10;
+                                                                                
+   P  = 1;
+   Q  = num_procs;
+   R  = 1;
+                                                                                
+   /*-----------------------------------------------------------
+    * Parse command line
+    *-----------------------------------------------------------*/
+   arg_index = 0;
+   while (arg_index < argc)
+   {
+      if ( strcmp(argv[arg_index], "-n") == 0 )
+      {
+         arg_index++;
+         nx = atoi(argv[arg_index++]);
+         ny = atoi(argv[arg_index++]);
+         nz = atoi(argv[arg_index++]);
+      }
+      else if ( strcmp(argv[arg_index], "-P") == 0 )
+      {
+         arg_index++;
+         P  = atoi(argv[arg_index++]);
+         Q  = atoi(argv[arg_index++]);
+         R  = atoi(argv[arg_index++]);
+      }
+      else
+      {
+         arg_index++;
+      }
+   }
+                                                                                
+   /* compute p,q,r from P,Q,R and myid */
+   p = myid % P;
+   q = (( myid - p)/P) % Q;
+   r = ( myid - p - P*q)/( P*Q );
+                                                                                
+   /*-----------------------------------------------------------
+    * Generate the coordinates
+    *-----------------------------------------------------------*/
+                                                                                
+   coorddim = 3;
+   if (nx<2) coorddim--;
+   if (ny<2) coorddim--;
+   if (nz<2) coorddim--;
+                                                                                
+   if (coorddim>0)
+      coordinates = GenerateCoordinates (hypre_MPI_COMM_WORLD,
+                                         nx, ny, nz, P, Q, R, p, q, r, coorddim);
+   else
+      coordinates=NULL;
+                                                                                
+   *coorddim_ptr = coorddim;
+   *coord_ptr = coordinates;
+   return (0);
+}
+
+
+/* begin lobpcg */
+
+/*----------------------------------------------------------------------
+ *  * Build standard 7-point laplacian in 3D.
+ *   *----------------------------------------------------------------------*/
+
+HYPRE_Int
 BuildParIsoLaplacian( HYPRE_Int argc, char** argv, HYPRE_ParCSRMatrix *A_ptr )
 {
 
@@ -4775,16 +6823,16 @@ BuildParIsoLaplacian( HYPRE_Int argc, char** argv, HYPRE_ParCSRMatrix *A_ptr )
    HYPRE_Int arg_index;
 
    /*-----------------------------------------------------------
-    * Initialize some stuff
-    *-----------------------------------------------------------*/
+ *     * Initialize some stuff
+ *         *-----------------------------------------------------------*/
 
    hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
    hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &myid );
 
    /*-----------------------------------------------------------
-    * Set defaults
-    *-----------------------------------------------------------*/
- 
+ *     * Set defaults
+ *         *-----------------------------------------------------------*/
+
    P  = 1;
    Q  = num_procs;
    R  = 1;
@@ -4815,9 +6863,9 @@ BuildParIsoLaplacian( HYPRE_Int argc, char** argv, HYPRE_ParCSRMatrix *A_ptr )
    }
 
    /*-----------------------------------------------------------
-    * Print driver parameters
-    *-----------------------------------------------------------*/
- 
+ *     * Print driver parameters
+ *         *-----------------------------------------------------------*/
+
    if (myid == 0)
    {
       hypre_printf("  Laplacian:\n");
@@ -4827,8 +6875,8 @@ BuildParIsoLaplacian( HYPRE_Int argc, char** argv, HYPRE_ParCSRMatrix *A_ptr )
    }
 
    /*-----------------------------------------------------------
-    * Set up the grid structure
-    *-----------------------------------------------------------*/
+ *     * Set up the grid structure
+ *         *-----------------------------------------------------------*/
 
    /* compute p,q,r from P,Q,R and myid */
    p = myid % P;
@@ -4836,9 +6884,9 @@ BuildParIsoLaplacian( HYPRE_Int argc, char** argv, HYPRE_ParCSRMatrix *A_ptr )
    r = ( myid - p - P*q)/( P*Q );
 
    /*-----------------------------------------------------------
-    * Generate the matrix 
-    *-----------------------------------------------------------*/
- 
+ *     * Generate the matrix 
+ *         *-----------------------------------------------------------*/
+
    values = hypre_CTAlloc(HYPRE_Real, 4);
 
    values[1] = -cx;
@@ -4859,8 +6907,8 @@ BuildParIsoLaplacian( HYPRE_Int argc, char** argv, HYPRE_ParCSRMatrix *A_ptr )
       values[0] += 2.0*cz;
    }
 
-   A = (HYPRE_ParCSRMatrix) GenerateLaplacian(hypre_MPI_COMM_WORLD, 
-		nx, ny, nz, P, Q, R, p, q, r, values);
+   A = (HYPRE_ParCSRMatrix) GenerateLaplacian(hypre_MPI_COMM_WORLD,
+                nx, ny, nz, P, Q, R, p, q, r, values);
 
    hypre_TFree(values);
 
@@ -4870,3 +6918,4 @@ BuildParIsoLaplacian( HYPRE_Int argc, char** argv, HYPRE_ParCSRMatrix *A_ptr )
 }
 
 /* end lobpcg */
+
diff --git a/src/test/runcheck.sh b/src/test/runcheck.sh
new file mode 100755
index 0000000..87afb1a
--- /dev/null
+++ b/src/test/runcheck.sh
@@ -0,0 +1,89 @@
+#!/bin/sh
+#BHEADER**********************************************************************
+# Copyright (c) 2008,  Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+# This file is part of HYPRE.  See file COPYRIGHT for details.
+#
+# HYPRE is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+#
+# $Revision$
+#EHEADER**********************************************************************
+# example call
+# ./runcheck.sh fname.out fname.saved 1.0e-6
+
+FNAME=$1
+SNAME=$2
+CONVTOL=$3
+
+if [ x$CONVTOL = "x" ]; 
+then
+    CONVTOL=0.0
+fi
+#echo "runcheck tol = $CONVTOL"
+
+PASSFAIL=$(awk -v filename="$SNAME" 'BEGIN{{FS=" "}
+
+   key = 0;
+   
+   # Read saved file data into array
+   while (getline < filename)
+   {
+      if(NF > 0 && substr($1,1,1) !~ /#/)
+      {            
+         saved_array[++key]=$NF;   
+      }
+   }
+   close(filename);
+  
+   # read out file and compare
+   filename="'"$FNAME"'"
+   tol = "'"$CONVTOL"'" + 0. # adding zero is necessary to convert from string to number
+   key=0;
+   pass=1;
+   while (getline < filename)
+   {
+      if(NF > 0 && substr($1,1,1) !~ /#/)
+      {
+         # get corresponding value in saved array
+         val = saved_array[++key];
+         
+         # floating point field comparison         
+         if($NF != int($NF))
+         {
+            tau = (val - $NF)/val;
+            # get absolute value of tau
+            tau = tau < 0 ? -tau : tau;
+            if(tau < tol)
+               #print "PASSED"
+               continue;
+            else
+               pass=0;
+               #print "FAILED"
+         }
+         else # integer comparison
+         {
+            tau = val - $NF;
+            # get absolute value of tau
+            tau = tau < 0 ? -tau : tau;
+            # get ceiling of tol*val (= max allowed change)
+            gamma = int(1.0 + tol*val);
+            if(tau <= gamma)
+               #print "PASSED"
+               continue;
+            else
+               pass=0;
+               #print "FAILED"           
+         }
+      }
+   }
+   if(pass != 1)
+      print "FAILED"
+}')
+
+if [ x$PASSFAIL != "x" ]; 
+then
+#    echo $PASSFAIL
+     diff -U3 -bI"time" $SNAME $FNAME >&2
+fi
diff --git a/src/test/runtest.sh b/src/test/runtest.sh
index 163d08a..17c439e 100755
--- a/src/test/runtest.sh
+++ b/src/test/runtest.sh
@@ -3,14 +3,17 @@
 # global variables
 BatchMode=0
 NoRun=0
-JobCheckInterval=10        #sleep time between jobs finished check
+JobCheckInterval=10        # sleep time between jobs finished check
 InputString=""
+RunPrefix=`type -p mpirun`
+RunPrefix="$RunPrefix -np"
 RunString=""
 RunEcho=""
-ExecFileNames=""           #string of executable file names used
-TestDirNames=""            #string of names of TEST_* directories used
+ExecFileNames=""           # string of executable file names used
+TestDirNames=""            # string of names of TEST_* directories used
 HOST=`hostname`
-NumThreads=0               #number of OpenMP threads to use if > 0
+NumThreads=0               # number of OpenMP threads to use if > 0
+Valgrind=""                # string to add to MpirunString when using valgrind
 
 function usage
 {
@@ -22,7 +25,10 @@ function usage
    printf "\n"
    printf " with options:\n"
    printf "    -h|-help       prints this usage information and exits\n"
+   printf "    -mpi <prefix>  MPI run prefix; default is 'mpirun -np'\n"
    printf "    -nthreads <n>  use 'n' OpenMP threads\n"
+   printf "    -tol <tol>     use relative tolerance 'tol' to compare numeric test values\n"
+   printf "    -valgrind      use valgrind memory checker\n"
    printf "    -n|-norun      turn off execute mode, echo what would be run\n"
    printf "    -t|-trace      echo each command\n"
    printf "    -D <var>       define <var> when running tests\n"
@@ -52,8 +58,9 @@ function usage
 function MpirunString
 {
    case $HOST in
-      *bgl*) shift
+      *bgl*)
          BatchMode=1
+         shift
          MY_NUM_TASKS=$1  
          MY_EXECUTE_DIR=`pwd`
          MY_EXECUTE_JOB=`pwd`/$EXECFILE
@@ -63,7 +70,8 @@ function MpirunString
          RunString="mpirun -verbose 1 -np $MY_NUM_TASKS -exe $MY_EXECUTE_JOB"
          RunString="${RunString} -cwd $MY_EXECUTE_DIR -args \" $MY_ARGS \" "
          ;;
-      up*) CPUS_PER_NODE=8
+      up*)
+         CPUS_PER_NODE=8
          POE_NUM_PROCS=$2
          POE_NUM_NODES=`expr $POE_NUM_PROCS + $CPUS_PER_NODE - 1`
          POE_NUM_NODES=`expr $POE_NUM_NODES / $CPUS_PER_NODE`
@@ -74,18 +82,8 @@ function MpirunString
          # RunString="${RunString} -nodes $POE_NUM_NODES $MY_ARGS"
          RunString="poe $MY_ARGS -rmpool pdebug -procs $POE_NUM_PROCS -nodes $POE_NUM_NODES"
          ;;
-      *dawn*) shift
-         BatchMode=1
-         MY_NUM_TASKS=$1  
-         MY_EXECUTE_DIR=`pwd`
-         MY_EXECUTE_JOB=`pwd`/$EXECFILE
+      rzzeus*|rzmerl*|ansel*|aztec*|cab*|sierra*|vulcan*)
          shift
-         shift
-         MY_ARGS="$*"
-         RunString="mpirun -verbose 1 -np $MY_NUM_TASKS -exe $MY_EXECUTE_JOB"
-         RunString="${RunString} -cwd $MY_EXECUTE_DIR -args \" $MY_ARGS \" "
-         ;;
-      rzzeus*|rzmerl*|ansel*|aztec*|cab*|sierra*|vulcan*) shift
          if [ $NumThreads -gt 0 ] ; then
             export OMP_NUM_THREADS=$NumThreads
             RunString="srun -p pdebug -c $NumThreads -n$*"
@@ -93,19 +91,14 @@ function MpirunString
             RunString="srun -p pdebug -n$*"
          fi
          ;;
-      tux*) BatchMode=0
+      *)
+         shift
          if [ $NumThreads -gt 0 ] ; then
             export OMP_NUM_THREADS=$NumThreads
          fi
-         MACHINES_FILE="hostname"
-         if [ ! -f $MACHINES_FILE ] ; then
-            hostname > $MACHINES_FILE
-         fi
-         MPIRUN=`type mpirun|sed -e 's/^.* //'`
-         RunString="$MPIRUN -machinefile $MACHINES_FILE $*"
-         ;;
-      *) MPIRUN=`type mpirun|sed -e 's/^.* //'`
-         RunString="$MPIRUN $*"
+         RunString="$RunPrefix $1"
+         shift
+         RunString="$RunString $Valgrind $*"
          ;;
    esac
 }
@@ -171,35 +164,6 @@ function CalcProcs
    return 1
 }
 
-# determine if HOST machine can process batch queues
-#    set to run in debug pool unless batch MUST be used.
-function CheckBatch
-{
-   case $HOST in
-      alc*) BATCH_MODE=0
-         ;;
-      peng*) BATCH_MODE=0
-         ;;
-      thun*) BATCH_MODE=0
-         ;;
-      *bgl*) BATCH_MODE=1
-         ;;
-      up*) BATCH_MODE=0
-         ;;
-      *dawn*) BATCH_MODE=1
-         ;;
-      vert*) BATCH_MODE=0
-         ;;
-      hera*) BATCH_MODE=0
-         ;;
-      *zeus*) BATCH_MODE=0
-         ;;
-      *) BATCH_MODE=0
-         ;;
-   esac
-   return $BATCH_MODE
-}
-
 # check the path to the executable if the executable exists; save the name to
 # ExecFileNames
 function CheckPath
@@ -303,12 +267,12 @@ function ExecuteJobs
             ;; 
 
          *mpirun*)
-            RunCmd=`echo $InputLine| sed -e 's/^[ \t]*mpirun[ \t]*//'` 
-            RunCmd=`echo $RunCmd | sed -e 's/[ \t]*>.*$//'`
-            OutFile=`echo $InputLine | sed -e 's/^.*>//'`
-            OutFile=`echo $OutFile | sed -e 's/ //g'`
-            ErrFile=`echo $OutFile | sed -e 's/\.out\./.err./'`
-            RunName=`echo $OutFile | sed -e 's/\.out.*$//'`
+            RunCmd=`echo $InputLine| sed -e 's/^[ \t]*mpirun[ \t]*//'` # remove 'mpirun'
+            RunCmd=`echo $RunCmd | sed -e 's/[ \t]*>.*$//'`            # remove output redirect
+            OutFile=`echo $InputLine | sed -e 's/^.*>//'`           # set output file
+            OutFile=`echo $OutFile | sed -e 's/ //g'`               # remove extra space
+            ErrFile=`echo $OutFile | sed -e 's/\.out\./.err./'`  # set error file
+            RunName=`echo $OutFile | sed -e 's/\.out.*$//'`   # set test run name
             CheckPath $RunCmd               # check path to executable
             if [ "$?" -gt 0 ] ; then
                cat >> $RunName.err <<- EOF
@@ -390,10 +354,11 @@ function ExecuteTest
    StartDir=$1
    WorkingDir=$2
    InputFile=$3
+   CONVTOL=$4
    SavePWD=`pwd`
    cd $WorkingDir
    (cat $InputFile.err.* > $InputFile.err)
-   (./$InputFile.sh     >> $InputFile.err 2>> $InputFile.err)
+   (./$InputFile.sh $CONVTOL   >> $InputFile.err 2>> $InputFile.err)
    cd $SavePWD
 }
 
@@ -422,7 +387,7 @@ function PostProcess
 }
 
                
-# removes executable and hostname files from all TEST_* directories
+# removes executables from all TEST_* directories
 function CleanUp
 {
    if [ "$BatchMode" -eq 0 ] ; then
@@ -433,13 +398,8 @@ function CleanUp
             ExecuteFile=$i/$j
             if [ -x $ExecuteFile ] ; then
                rm -f $ExecuteFile
-               rm -f hostname
             fi
          done
-         ExecuteFile=$i/hostname
-         if [ -f $ExecuteFile ] ; then
-            rm -f $ExecuteFile
-         fi
          case $i in
             TEST_examples)
                rm -f ex? ex?? ex??f
@@ -453,8 +413,6 @@ function StartCrunch
 {
    rm -f ~/insure.log*
 
-   CheckBatch
-   BatchMode=$?
    ExecuteJobs "$@"
    ExecuteTest "$@"
    PostProcess "$@"
@@ -464,7 +422,8 @@ function StartCrunch
 #==========================================================================
 
 # main
-
+# Set default check tolerance
+CONVTOL=0.0
 while [ "$*" ]
 do
    case $1 in
@@ -472,11 +431,25 @@ do
          usage
          exit
          ;;
+      -mpi)
+         shift
+         MPIRunPrefix=$1
+         shift
+         ;;
       -nthreads)
          shift
          NumThreads=$1
          shift
          ;;
+      -tol)
+         shift
+         CONVTOL=$1
+         shift
+         ;;
+      -valgrind)
+         shift
+         Valgrind="valgrind -q --suppressions=`pwd`/runtest.valgrind --leak-check=yes --track-origins=yes"
+         ;;
       -n|-norun)
          NoRun=1
          RunEcho="echo"
@@ -513,7 +486,14 @@ do
                      ;;
                esac
                if [ -r $DirPart/$FilePart.jobs ] ; then
-                  StartCrunch $CurDir $DirPart $FilePart
+
+                  # Check for an mpirun routine
+                  if [ x$MPIRunprefix != "x" ]
+                  then
+                     RunPrefix=$MPIRunPrefix
+                  fi
+
+                  StartCrunch $CurDir $DirPart $FilePart $CONVTOL
                else
                   printf "%s: test command file %s/%s.jobs does not exist\n" \
                      $0 $DirPart $FilePart
diff --git a/src/test/runtest.valgrind b/src/test/runtest.valgrind
new file mode 100644
index 0000000..e69de29
diff --git a/src/test/sstruct.c b/src/test/sstruct.c
index 732b36e..b5420f2 100644
--- a/src/test/sstruct.c
+++ b/src/test/sstruct.c
@@ -201,6 +201,9 @@ typedef struct
    ProblemPartData *pdata;
    HYPRE_Int        max_boxsize;
 
+   /* for GridSetNumGhost */
+   HYPRE_Int       *numghost;
+
    HYPRE_Int        nstencils;
    HYPRE_Int       *stencil_sizes;
    Index          **stencil_offsets;
@@ -476,6 +479,7 @@ ReadData( char         *filename,
     *-----------------------------------------------------------*/
 
    data.max_boxsize = 0;
+   data.numghost = NULL;
    data.nstencils = 0;
    data.rhs_true = 0;
    data.fem_nvars = 0;
@@ -506,6 +510,13 @@ ReadData( char         *filename,
             data.nparts = strtol(sdata_ptr, &sdata_ptr, 10);
             data.pdata = hypre_CTAlloc(ProblemPartData, data.nparts);
          }
+         else if ( strcmp(key, "GridSetNumGhost:") == 0 )
+         {
+            // # GridSetNumGhost: numghost[2*ndim]
+            // GridSetNumGhost: [3 3 3 3]
+            data.numghost = hypre_CTAlloc(HYPRE_Int, 2*data.ndim);
+            SScanIntArray(sdata_ptr, &sdata_ptr, 2*data.ndim, data.numghost);
+         }
          else if ( strcmp(key, "GridSetExtents:") == 0 )
          {
             part = strtol(sdata_ptr, &sdata_ptr, 10);
@@ -2054,6 +2065,8 @@ DestroyData( ProblemData   data )
    }
    hypre_TFree(data.pdata);
 
+   hypre_TFree(data.numghost);
+
    if (data.nstencils > 0)
    {
       for (s = 0; s < data.nstencils; s++)
@@ -2243,6 +2256,7 @@ PrintUsage( char *progname,
       hypre_printf("  -crtdim <tdim>     : Struct- cyclic reduction tdim\n");
       hypre_printf("  -cri <ix> <iy> <iz>: Struct- cyclic reduction base_index\n");
       hypre_printf("  -crs <sx> <sy> <sz>: Struct- cyclic reduction base_stride\n");
+      hypre_printf("  -old_default: sets old BoomerAMG defaults, possibly better for 2D problems\n");
 
       /* begin lobpcg */
 
@@ -2375,6 +2389,7 @@ main( hypre_int argc,
    HYPRE_Int printLevel = 0;
    HYPRE_Int pcgIterations = 0;
    HYPRE_Int pcgMode = 0;
+   HYPRE_Int old_default = 0;
    HYPRE_Real tol = 1e-6;
    HYPRE_Real pcgTol = 1e-2;
    HYPRE_Real nonOrthF;
@@ -2685,6 +2700,11 @@ main( hypre_int argc,
          arg_index++;	      /* 0: zero, otherwise rhs */
          pcgMode = atoi(argv[arg_index++]);
       }
+      else if ( strcmp(argv[arg_index], "-old_default") == 0 )
+      {		 /* uses old BoomerAMG defaults */
+         arg_index++;
+         old_default = 1;
+      }
       else if ( strcmp(argv[arg_index], "-vout") == 0 )
       {			      /* lobpcg: print level */
          arg_index++;
@@ -2754,6 +2774,10 @@ main( hypre_int argc,
    hypre_BeginTiming(time_index);
 
    HYPRE_SStructGridCreate(hypre_MPI_COMM_WORLD, data.ndim, data.nparts, &grid);
+   if (data.numghost != NULL)
+   {
+      HYPRE_SStructGridSetNumGhost(grid, data.numghost);
+   }
    for (part = 0; part < data.nparts; part++)
    {
       pdata = data.pdata[part];
@@ -4183,7 +4207,7 @@ main( hypre_int argc,
       {
          /* use BoomerAMG as preconditioner */
          HYPRE_BoomerAMGCreate(&par_precond); 
-         HYPRE_BoomerAMGSetCoarsenType(par_precond, 6);
+         if (old_default) HYPRE_BoomerAMGSetOldDefault(par_precond);
          HYPRE_BoomerAMGSetStrongThreshold(par_precond, 0.25);
          HYPRE_BoomerAMGSetTol(par_precond, 0.0);
          HYPRE_BoomerAMGSetPrintLevel(par_precond, 1);
@@ -4358,7 +4382,7 @@ main( hypre_int argc,
       {
          /* use BoomerAMG as preconditioner */
          HYPRE_BoomerAMGCreate(&par_precond); 
-         HYPRE_BoomerAMGSetCoarsenType(par_precond, 6);
+         if (old_default) HYPRE_BoomerAMGSetOldDefault(par_precond);
          HYPRE_BoomerAMGSetStrongThreshold(par_precond, 0.25);
          HYPRE_BoomerAMGSetTol(par_precond, 0.0);
          HYPRE_BoomerAMGSetPrintLevel(par_precond, 1);
@@ -4522,7 +4546,7 @@ main( hypre_int argc,
       {
          /* use BoomerAMG as preconditioner */
          HYPRE_BoomerAMGCreate(&par_precond); 
-         HYPRE_BoomerAMGSetCoarsenType(par_precond, 6);
+         if (old_default) HYPRE_BoomerAMGSetOldDefault(par_precond);
          HYPRE_BoomerAMGSetStrongThreshold(par_precond, 0.25);
          HYPRE_BoomerAMGSetTol(par_precond, 0.0);
          HYPRE_BoomerAMGSetPrintLevel(par_precond, 1);
@@ -4689,7 +4713,7 @@ main( hypre_int argc,
       {
          /* use BoomerAMG as preconditioner */
          HYPRE_BoomerAMGCreate(&par_precond); 
-         HYPRE_BoomerAMGSetCoarsenType(par_precond, 6);
+         if (old_default) HYPRE_BoomerAMGSetOldDefault(par_precond);
          HYPRE_BoomerAMGSetStrongThreshold(par_precond, 0.25);
          HYPRE_BoomerAMGSetTol(par_precond, 0.0);
          HYPRE_BoomerAMGSetPrintLevel(par_precond, 1);
@@ -4751,7 +4775,7 @@ main( hypre_int argc,
       {
          /* use BoomerAMG as preconditioner */
          HYPRE_BoomerAMGCreate(&par_precond); 
-         HYPRE_BoomerAMGSetCoarsenType(par_precond, 6);
+         if (old_default) HYPRE_BoomerAMGSetOldDefault(par_precond);
          HYPRE_BoomerAMGSetStrongThreshold(par_precond, 0.25);
          HYPRE_BoomerAMGSetTol(par_precond, 0.0);
          HYPRE_BoomerAMGSetPrintLevel(par_precond, 1);
diff --git a/src/test/sstruct.in.README b/src/test/sstruct.in.README
index 1a8d66a..2f5cd8c 100644
--- a/src/test/sstruct.in.README
+++ b/src/test/sstruct.in.README
@@ -20,6 +20,9 @@
 # GridCreate: ndim nparts
 GridCreate: 2 2
 
+## GridSetNumGhost: numghost[2*ndim]
+#GridSetNumGhost: [3 3 3 3]
+
 # GridSetExtents: part ilower(ndim) iupper(ndim)
 GridSetExtents: 0 (0- 0-) (4+ 4+)
 
diff --git a/src/test/struct.c b/src/test/struct.c
index 856cb62..9187bc0 100644
--- a/src/test/struct.c
+++ b/src/test/struct.c
@@ -85,8 +85,9 @@ main( hypre_int argc,
 
    /*HYPRE_Real          dxyz[3];*/
 
+   HYPRE_Int           num_ghost[6]   = {0, 0, 0, 0, 0, 0};
    HYPRE_Int           A_num_ghost[6] = {0, 0, 0, 0, 0, 0};
-   HYPRE_Int           v_num_ghost[3] = {0,0,0};
+   HYPRE_Int           v_num_ghost[6] = {0, 0, 0, 0, 0, 0};
                      
    HYPRE_StructMatrix  A;
    HYPRE_StructVector  b;
@@ -256,11 +257,12 @@ main( hypre_int argc,
    read_x0fromfile_index = argc;
    sum = 0;
 
-   /* ghosts for the building of matrix: default  */
-   for (i = 0; i < dim; i++)
+   /* ghost defaults */
+   for (i = 0; i < 2*dim; i++)
    {
-      A_num_ghost[2*i] = 1;
-      A_num_ghost[2*i + 1] = 1;
+      num_ghost[i]   = 1;
+      A_num_ghost[i] = num_ghost[i];
+      v_num_ghost[i] = num_ghost[i];
    }
 
    /*-----------------------------------------------------------
@@ -966,18 +968,13 @@ main( hypre_int argc,
             HYPRE_StructGridSetExtents(grid, ilower[ib], iupper[ib]);
          }
          HYPRE_StructGridSetPeriodic(grid, periodic);
+         HYPRE_StructGridSetNumGhost(grid, num_ghost);
          HYPRE_StructGridAssemble(grid);
 
          /*-----------------------------------------------------------
           * Set up the matrix structure
           *-----------------------------------------------------------*/
 
-         for (i = 0; i < dim; i++)
-         {
-            A_num_ghost[2*i] = 1;
-            A_num_ghost[2*i + 1] = 1;
-         }
-
          HYPRE_StructMatrixCreate(hypre_MPI_COMM_WORLD, grid, stencil, &A);
          if ( solver_id == 3 || solver_id == 4 ||
               solver_id == 13 || solver_id == 14 )
@@ -1012,7 +1009,6 @@ main( hypre_int argc,
             }
          }
          HYPRE_StructMatrixSetSymmetric(A, sym);
-         HYPRE_StructMatrixSetNumGhost(A, A_num_ghost);
          HYPRE_StructMatrixInitialize(A);
 
          /*-----------------------------------------------------------
@@ -1075,13 +1071,6 @@ main( hypre_int argc,
                hypre_printf("\ninitial rhs from file prefix :%s\n",
                             argv[read_rhsfromfile_index]);
 
-               /* ghost selection for vector  */
-               for (i = 0; i < dim; i++)
-               {
-                  v_num_ghost[2*i] = 1;
-                  v_num_ghost[2*i + 1] = 1;
-               }
-           
                b = (HYPRE_StructVector)
                   hypre_StructVectorRead(hypre_MPI_COMM_WORLD,
                                          argv[read_rhsfromfile_index],
@@ -1099,7 +1088,6 @@ main( hypre_int argc,
                HYPRE_StructMatrixCreate(hypre_MPI_COMM_WORLD,
                                         readgrid, stencil, &A);
                HYPRE_StructMatrixSetSymmetric(A, 1);
-               HYPRE_StructMatrixSetNumGhost(A, A_num_ghost);
                HYPRE_StructMatrixInitialize(A);
 
                /*-----------------------------------------------------------
@@ -1125,13 +1113,6 @@ main( hypre_int argc,
                hypre_printf("\ninitial x0 from file prefix :%s\n",
                             argv[read_x0fromfile_index]);
 
-               /* ghost selection for vector  */
-               for (i = 0; i < dim; i++)
-               {
-                  v_num_ghost[2*i] = 1;
-                  v_num_ghost[2*i + 1] = 1;
-               }
-  
                x = (HYPRE_StructVector)
                   hypre_StructVectorRead(hypre_MPI_COMM_WORLD,
                                          argv[read_x0fromfile_index],v_num_ghost);
@@ -1148,7 +1129,6 @@ main( hypre_int argc,
                HYPRE_StructMatrixCreate(hypre_MPI_COMM_WORLD,
                                         readgrid, stencil, &A);
                HYPRE_StructMatrixSetSymmetric(A, 1);
-               HYPRE_StructMatrixSetNumGhost(A, A_num_ghost);
                HYPRE_StructMatrixInitialize(A);
 
                /*-----------------------------------------------------------
@@ -1176,13 +1156,6 @@ main( hypre_int argc,
                hypre_printf("\ninitial x0  from file prefix :%s\n",
                             argv[read_x0fromfile_index]);
 
-               /* ghost selection for vector  */
-               for (i = 0; i < dim; i++)
-               {
-                  v_num_ghost[2*i] = 1;
-                  v_num_ghost[2*i + 1] = 1;
-               }
-  
                b = (HYPRE_StructVector)
                   hypre_StructVectorRead(hypre_MPI_COMM_WORLD,
                                          argv[read_rhsfromfile_index],
@@ -1199,7 +1172,6 @@ main( hypre_int argc,
                HYPRE_StructMatrixCreate(hypre_MPI_COMM_WORLD,
                                         readgrid, stencil, &A);
                HYPRE_StructMatrixSetSymmetric(A, 1);
-               HYPRE_StructMatrixSetNumGhost(A, A_num_ghost);
                HYPRE_StructMatrixInitialize(A);
 
                /*-----------------------------------------------------------
@@ -1222,12 +1194,6 @@ main( hypre_int argc,
          {   
             hypre_printf("\nreading matrix from file:%s\n",
                          argv[read_fromfile_index]);
-            /* ghost selection for reading the matrix  */
-            for (i = 0; i < dim; i++)
-            {
-               A_num_ghost[2*i] = 1;
-               A_num_ghost[2*i + 1] = 1;
-            }
 
             A = (HYPRE_StructMatrix)
                hypre_StructMatrixRead(hypre_MPI_COMM_WORLD,
@@ -1242,13 +1208,6 @@ main( hypre_int argc,
                hypre_printf("\ninitial rhs from file prefix :%s\n",
                             argv[read_rhsfromfile_index]);
 
-               /* ghost selection for vector  */
-               for (i = 0; i < dim; i++)
-               {
-                  v_num_ghost[2*i] = 1;
-                  v_num_ghost[2*i + 1] = 1;
-               }
-  
                b = (HYPRE_StructVector)
                   hypre_StructVectorRead(hypre_MPI_COMM_WORLD,
                                          argv[read_rhsfromfile_index],
@@ -1266,13 +1225,6 @@ main( hypre_int argc,
                hypre_printf("\ninitial x0 from file prefix :%s\n",
                             argv[read_x0fromfile_index]);
 
-               /* ghost selection for vector  */
-               for (i = 0; i < dim; i++)
-               {
-                  v_num_ghost[2*i] = 1;
-                  v_num_ghost[2*i + 1] = 1;
-               }
-  
                x = (HYPRE_StructVector)
                   hypre_StructVectorRead(hypre_MPI_COMM_WORLD,
                                          argv[read_x0fromfile_index],
@@ -1340,7 +1292,7 @@ main( hypre_int argc,
          HYPRE_StructSMGCreate(hypre_MPI_COMM_WORLD, &solver);
          HYPRE_StructSMGSetMemoryUse(solver, 0);
          HYPRE_StructSMGSetMaxIter(solver, 50);
-         HYPRE_StructSMGSetTol(solver, 1.0e-06);
+         HYPRE_StructSMGSetTol(solver, tol);
          HYPRE_StructSMGSetRelChange(solver, 0);
          HYPRE_StructSMGSetNumPreRelax(solver, n_pre);
          HYPRE_StructSMGSetNumPostRelax(solver, n_post);
@@ -1393,7 +1345,7 @@ main( hypre_int argc,
          HYPRE_StructPFMGCreate(hypre_MPI_COMM_WORLD, &solver);
          /*HYPRE_StructPFMGSetMaxLevels( solver, 9 );*/
          HYPRE_StructPFMGSetMaxIter(solver, 200);
-         HYPRE_StructPFMGSetTol(solver, 1.0e-06);
+         HYPRE_StructPFMGSetTol(solver, tol);
          HYPRE_StructPFMGSetRelChange(solver, 0);
          HYPRE_StructPFMGSetRAPType(solver, rap);
          HYPRE_StructPFMGSetRelaxType(solver, relax);
@@ -1455,7 +1407,7 @@ main( hypre_int argc,
          HYPRE_StructSparseMSGCreate(hypre_MPI_COMM_WORLD, &solver);
          HYPRE_StructSparseMSGSetMaxIter(solver, 50);
          HYPRE_StructSparseMSGSetJump(solver, jump);
-         HYPRE_StructSparseMSGSetTol(solver, 1.0e-06);
+         HYPRE_StructSparseMSGSetTol(solver, tol);
          HYPRE_StructSparseMSGSetRelChange(solver, 0);
          HYPRE_StructSparseMSGSetRelaxType(solver, relax);
          if (usr_jacobi_weight)
@@ -1500,7 +1452,7 @@ main( hypre_int argc,
 
          HYPRE_StructJacobiCreate(hypre_MPI_COMM_WORLD, &solver);
          HYPRE_StructJacobiSetMaxIter(solver, 100);
-         HYPRE_StructJacobiSetTol(solver, 1.0e-06);
+         HYPRE_StructJacobiSetTol(solver, tol);
          HYPRE_StructJacobiSetup(solver, A, b, x);
 
          hypre_EndTiming(time_index);
@@ -1534,7 +1486,7 @@ main( hypre_int argc,
 
          HYPRE_StructPCGCreate(hypre_MPI_COMM_WORLD, &solver);
          HYPRE_PCGSetMaxIter( (HYPRE_Solver)solver, 100 );
-         HYPRE_PCGSetTol( (HYPRE_Solver)solver, 1.0e-06 );
+         HYPRE_PCGSetTol( (HYPRE_Solver)solver, tol );
          HYPRE_PCGSetTwoNorm( (HYPRE_Solver)solver, 1 );
          HYPRE_PCGSetRelChange( (HYPRE_Solver)solver, 0 );
          HYPRE_PCGSetPrintLevel( (HYPRE_Solver)solver, 1 );
@@ -2150,7 +2102,7 @@ main( hypre_int argc,
          HYPRE_StructHybridCreate(hypre_MPI_COMM_WORLD, &solver);
          HYPRE_StructHybridSetDSCGMaxIter(solver, 100);
          HYPRE_StructHybridSetPCGMaxIter(solver, 100);
-         HYPRE_StructHybridSetTol(solver, 1.0e-06);
+         HYPRE_StructHybridSetTol(solver, tol);
          /*HYPRE_StructHybridSetPCGAbsoluteTolFactor(solver, 1.0e-200);*/
          HYPRE_StructHybridSetConvergenceTol(solver, cf_tol);
          HYPRE_StructHybridSetTwoNorm(solver, 1);
@@ -2277,7 +2229,7 @@ main( hypre_int argc,
          HYPRE_StructGMRESCreate(hypre_MPI_COMM_WORLD, &solver);
          HYPRE_GMRESSetKDim( (HYPRE_Solver) solver, 5 );
          HYPRE_GMRESSetMaxIter( (HYPRE_Solver)solver, 100 );
-         HYPRE_GMRESSetTol( (HYPRE_Solver)solver, 1.0e-06 );
+         HYPRE_GMRESSetTol( (HYPRE_Solver)solver, tol );
          HYPRE_GMRESSetRelChange( (HYPRE_Solver)solver, 0 );
          HYPRE_GMRESSetPrintLevel( (HYPRE_Solver)solver, 1 );
          HYPRE_GMRESSetLogging( (HYPRE_Solver)solver, 1 );
@@ -2423,7 +2375,7 @@ main( hypre_int argc,
 
          HYPRE_StructBiCGSTABCreate(hypre_MPI_COMM_WORLD, &solver);
          HYPRE_BiCGSTABSetMaxIter( (HYPRE_Solver)solver, 100 );
-         HYPRE_BiCGSTABSetTol( (HYPRE_Solver)solver, 1.0e-06 );
+         HYPRE_BiCGSTABSetTol( (HYPRE_Solver)solver, tol );
          HYPRE_BiCGSTABSetPrintLevel( (HYPRE_Solver)solver, 1 );
          HYPRE_BiCGSTABSetLogging( (HYPRE_Solver)solver, 1 );
 
@@ -2568,7 +2520,7 @@ main( hypre_int argc,
          HYPRE_StructLGMRESCreate(hypre_MPI_COMM_WORLD, &solver);
          HYPRE_LGMRESSetKDim( (HYPRE_Solver) solver, 5 );
          HYPRE_LGMRESSetMaxIter( (HYPRE_Solver)solver, 100 );
-         HYPRE_LGMRESSetTol( (HYPRE_Solver)solver, 1.0e-06 );
+         HYPRE_LGMRESSetTol( (HYPRE_Solver)solver, tol );
          HYPRE_LGMRESSetPrintLevel( (HYPRE_Solver)solver, 1 );
          HYPRE_LGMRESSetLogging( (HYPRE_Solver)solver, 1 );
 
@@ -2660,7 +2612,7 @@ main( hypre_int argc,
          HYPRE_StructFlexGMRESCreate(hypre_MPI_COMM_WORLD, &solver);
          HYPRE_FlexGMRESSetKDim( (HYPRE_Solver) solver, 5 );
          HYPRE_FlexGMRESSetMaxIter( (HYPRE_Solver)solver, 100 );
-         HYPRE_FlexGMRESSetTol( (HYPRE_Solver)solver, 1.0e-06 );
+         HYPRE_FlexGMRESSetTol( (HYPRE_Solver)solver, tol );
          HYPRE_FlexGMRESSetPrintLevel( (HYPRE_Solver)solver, 1 );
          HYPRE_FlexGMRESSetLogging( (HYPRE_Solver)solver, 1 );
 
diff --git a/src/utilities/_hypre_utilities.h b/src/utilities/_hypre_utilities.h
index 785595b..c21b717 100644
--- a/src/utilities/_hypre_utilities.h
+++ b/src/utilities/_hypre_utilities.h
@@ -64,6 +64,10 @@ extern "C" {
 #define hypre_min(a,b)  (((a)<(b)) ? (a) : (b))
 #endif
 
+#ifndef hypre_abs
+#define hypre_abs(a)  (((a)>0) ? (a) : -(a))
+#endif
+
 #ifndef hypre_round
 #define hypre_round(x)  ( ((x) < 0.0) ? ((HYPRE_Int)(x - 0.5)) : ((HYPRE_Int)(x + 0.5)) )
 #endif
@@ -677,9 +681,6 @@ HYPRE_Int hypre_PrintTiming( const char *heading , MPI_Comm comm );
 extern "C" {
 #endif
 
-#define LIST_HEAD -1
-#define LIST_TAIL -2
-
 struct double_linked_list
 {
        HYPRE_Int                        data;
@@ -800,6 +801,45 @@ void hypre_error_handler(const char *filename, HYPRE_Int line, HYPRE_Int ierr, c
 
 #endif
 
+/*BHEADER**********************************************************************
+ * Copyright (c) 2008,  Lawrence Livermore National Security, LLC.
+ * Produced at the Lawrence Livermore National Laboratory.
+ * This file is part of HYPRE.  See file COPYRIGHT for details.
+ *
+ * HYPRE is free software; you can redistribute it and/or modify it under the
+ * terms of the GNU Lesser General Public License (as published by the Free
+ * Software Foundation) version 2.1 dated February 1999.
+ *
+ * $Revision$
+ ***********************************************************************EHEADER*/
+
+/******************************************************************************
+ *
+ * Header file for Caliper instrumentation macros
+ *
+ *****************************************************************************/
+
+#ifndef CALIPER_INSTRUMENTATION_HEADER
+#define CALIPER_INSTRUMENTATION_HEADER
+
+#include "HYPRE_config.h"
+
+#ifdef HYPRE_USING_CALIPER
+
+#include <caliper/cali.h>
+
+#define HYPRE_ANNOTATION_BEGIN( str ) cali_begin_string_byname("hypre.kernel", str)
+#define HYPRE_ANNOTATION_END( str ) cali_end_byname("hypre.kernel")
+
+#else
+
+#define HYPRE_ANNOTATION_BEGIN( str ) 
+#define HYPRE_ANNOTATION_END( str ) 
+
+#endif
+
+#endif /* CALIPER_INSTRUMENTATION_HEADER */
+
 /*--------------------------------------------------------------------------
  * Other prototypes
  *--------------------------------------------------------------------------*/
diff --git a/src/utilities/amg_linklist.c b/src/utilities/amg_linklist.c
index 84ccd51..6a691ec 100644
--- a/src/utilities/amg_linklist.c
+++ b/src/utilities/amg_linklist.c
@@ -20,8 +20,8 @@
 
 #include "_hypre_utilities.h"
 
-#define LIST_HEAD -1
-#define LIST_TAIL -2
+#define hypre_LIST_HEAD -1
+#define hypre_LIST_TAIL -2
 
 
 /**************************************************************
@@ -114,13 +114,13 @@ hypre_remove_point(hypre_LinkList   *LoL_head_ptr,
          else if (list_ptr->head == index)      /* index is head of list */
          {
             list_ptr->head = lists[index];
-            where[lists[index]] = LIST_HEAD;
+            where[lists[index]] = hypre_LIST_HEAD;
             return;
          }
          else if (list_ptr->tail == index)      /* index is tail of list */
          {
             list_ptr->tail = where[index];
-            lists[where[index]] = LIST_TAIL;
+            lists[where[index]] = hypre_LIST_TAIL;
             return;
          }
          else                              /* index is in middle of list */
@@ -161,8 +161,8 @@ hypre_LinkList hypre_create_elt( HYPRE_Int Item )
        new_elt_ptr -> data = Item;
        new_elt_ptr -> next_elt = NULL;
        new_elt_ptr -> prev_elt = NULL;
-       new_elt_ptr -> head = LIST_TAIL;
-       new_elt_ptr -> tail = LIST_HEAD;
+       new_elt_ptr -> head = hypre_LIST_TAIL;
+       new_elt_ptr -> tail = hypre_LIST_HEAD;
     }
 
     return (new_elt_ptr);
@@ -196,8 +196,8 @@ hypre_enter_on_lists(hypre_LinkList   *LoL_head_ptr,
       new_ptr = hypre_create_elt(measure);
       new_ptr->head = index;
       new_ptr->tail = index;
-      lists[index] = LIST_TAIL;
-      where[index] = LIST_HEAD; 
+      lists[index] = hypre_LIST_TAIL;
+      where[index] = hypre_LIST_HEAD; 
       LoL_head = new_ptr;
       LoL_tail = new_ptr;
 
@@ -214,8 +214,8 @@ hypre_enter_on_lists(hypre_LinkList   *LoL_head_ptr,
          new_ptr = hypre_create_elt(measure);
          new_ptr->head = index;
          new_ptr->tail = index;
-         lists[index] = LIST_TAIL;
-         where[index] = LIST_HEAD;
+         lists[index] = hypre_LIST_TAIL;
+         where[index] = hypre_LIST_HEAD;
 
          if ( list_ptr->prev_elt != NULL)
          { 
@@ -241,7 +241,7 @@ hypre_enter_on_lists(hypre_LinkList   *LoL_head_ptr,
          old_tail = list_ptr->tail;
          lists[old_tail] = index;
          where[index] = old_tail;
-         lists[index] = LIST_TAIL;
+         lists[index] = hypre_LIST_TAIL;
          list_ptr->tail = index;
          return;
       }
@@ -252,8 +252,8 @@ hypre_enter_on_lists(hypre_LinkList   *LoL_head_ptr,
    new_ptr = hypre_create_elt(measure);   
    new_ptr->head = index;
    new_ptr->tail = index;
-   lists[index] = LIST_TAIL;
-   where[index] = LIST_HEAD;
+   lists[index] = hypre_LIST_TAIL;
+   where[index] = hypre_LIST_HEAD;
    LoL_tail->next_elt = new_ptr;
    new_ptr->prev_elt = LoL_tail;
    new_ptr->next_elt = NULL;
diff --git a/src/utilities/amg_linklist.h b/src/utilities/amg_linklist.h
index b456ba1..53911fe 100644
--- a/src/utilities/amg_linklist.h
+++ b/src/utilities/amg_linklist.h
@@ -29,9 +29,6 @@
 extern "C" {
 #endif
 
-#define LIST_HEAD -1
-#define LIST_TAIL -2
-
 struct double_linked_list
 {
        HYPRE_Int                        data;
diff --git a/src/distributed_ls/ParaSails/OrderStat.h b/src/utilities/caliper_instrumentation.h
similarity index 60%
copy from src/distributed_ls/ParaSails/OrderStat.h
copy to src/utilities/caliper_instrumentation.h
index e8455cf..0f3c071 100644
--- a/src/distributed_ls/ParaSails/OrderStat.h
+++ b/src/utilities/caliper_instrumentation.h
@@ -10,21 +10,29 @@
  * $Revision$
  ***********************************************************************EHEADER*/
 
-
-
-
 /******************************************************************************
  *
- * OrderStat.h header file.
+ * Header file for Caliper instrumentation macros
  *
  *****************************************************************************/
 
-#ifndef _ORDERSTAT_H
-#define _ORDERSTAT_H
+#ifndef CALIPER_INSTRUMENTATION_HEADER
+#define CALIPER_INSTRUMENTATION_HEADER
+
+#include "HYPRE_config.h"
+
+#ifdef HYPRE_USING_CALIPER
+
+#include <caliper/cali.h>
+
+#define HYPRE_ANNOTATION_BEGIN( str ) cali_begin_string_byname("hypre.kernel", str)
+#define HYPRE_ANNOTATION_END( str ) cali_end_byname("hypre.kernel")
+
+#else
 
-#include "_hypre_utilities.h"
+#define HYPRE_ANNOTATION_BEGIN( str ) 
+#define HYPRE_ANNOTATION_END( str ) 
 
-HYPRE_Real randomized_select(HYPRE_Real *a, HYPRE_Int p, HYPRE_Int r, HYPRE_Int i);
-void shell_sort(const HYPRE_Int n, HYPRE_Int x[]);
+#endif
 
-#endif /* _ORDERSTAT_H */
+#endif /* CALIPER_INSTRUMENTATION_HEADER */
diff --git a/src/utilities/general.h b/src/utilities/general.h
index 2397b7d..328a6c7 100644
--- a/src/utilities/general.h
+++ b/src/utilities/general.h
@@ -31,6 +31,10 @@
 #define hypre_min(a,b)  (((a)<(b)) ? (a) : (b))
 #endif
 
+#ifndef hypre_abs
+#define hypre_abs(a)  (((a)>0) ? (a) : -(a))
+#endif
+
 #ifndef hypre_round
 #define hypre_round(x)  ( ((x) < 0.0) ? ((HYPRE_Int)(x - 0.5)) : ((HYPRE_Int)(x + 0.5)) )
 #endif

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/hypre.git



More information about the debian-science-commits mailing list