[SCM] Gerris Flow Solver branch, upstream, updated. e8f73a07832050124d2b8bf6c6f35b33180e65a8
Stephane Popinet
popinet at users.sf.net
Tue Nov 24 12:24:46 UTC 2009
The following commit has been merged in the upstream branch:
commit ca1a86b1aeb09c2a90a91298bc05ec09264a46ce
Author: Stephane Popinet <popinet at users.sf.net>
Date: Thu Jul 23 14:55:44 2009 +1000
New function gfs_traverse_and_bc()
For overlapping computation and communications.
darcs-hash:20090723045544-d4795-30b937b83e2dcaa7eceeafee7ab36097ed41a0a1.gz
diff --git a/src/domain.c b/src/domain.c
index 89b225f..3eb7e1f 100644
--- a/src/domain.c
+++ b/src/domain.c
@@ -163,6 +163,8 @@ static void domain_write (GtsObject * o, FILE * fp)
if (domain->lambda.z != 1.)
fprintf (fp, "lz = %g ", domain->lambda.z);
fprintf (fp, "version = %d ", atoi (GFS_BUILD_VERSION));
+ if (!domain->overlap)
+ fputs ("overlap = 0 ", fp);
if (domain->max_depth_write > -2) {
GSList * i = domain->variables_io;
@@ -195,6 +197,7 @@ static void domain_read (GtsObject ** o, GtsFile * fp)
{GTS_STRING, "variables", TRUE},
{GTS_INT, "binary", TRUE},
{GTS_INT, "version", TRUE},
+ {GTS_INT, "overlap", TRUE},
{GTS_NONE}
};
gchar * variables = NULL;
@@ -215,6 +218,7 @@ static void domain_read (GtsObject ** o, GtsFile * fp)
var[7].data = &variables;
var[8].data = &domain->binary;
var[9].data = &domain->version;
+ var[10].data = &domain->overlap;
gts_file_assign_variables (fp, var);
if (fp->type == GTS_ERROR) {
g_free (variables);
@@ -578,6 +582,8 @@ static void domain_init (GfsDomain * domain)
domain->cell_init_data = domain;
domain->version = atoi (GFS_BUILD_VERSION);
+
+ domain->overlap = TRUE;
}
GfsDomainClass * gfs_domain_class (void)
@@ -839,8 +845,10 @@ typedef struct {
static void update_mpi_cell (FttCell * cell, TraverseData * p)
{
- (* p->func) (cell, p->data);
- cell->flags |= GFS_FLAG_USED;
+ if ((cell->flags & GFS_FLAG_USED) == 0) {
+ (* p->func) (cell, p->data);
+ cell->flags |= GFS_FLAG_USED;
+ }
}
static void update_other_cell (FttCell * cell, TraverseData * p)
@@ -868,14 +876,14 @@ static void update_mpi_boundaries (GfsBox * box, TraverseBcData * p)
b->type = GFS_BOUNDARY_CENTER_VARIABLE;
ftt_face_traverse_boundary (b->root, b->d,
FTT_PRE_ORDER, p->b.flags, p->b.max_depth,
- bc->homogeneous_bc, bc);
+ bc->bc, bc);
bc->v = p->b.v;
gfs_boundary_send (b);
}
}
}
-static void update_other_boundaries (GfsBox * box, BcData * p)
+static void update_other_homogeneous_boundaries (GfsBox * box, BcData * p)
{
FttDirection d;
for (d = 0; d < FTT_NEIGHBORS; d++)
@@ -931,7 +939,7 @@ void gfs_traverse_and_homogeneous_bc (GfsDomain * domain,
{
g_return_if_fail (domain != NULL);
- if (domain->pid < 0) {
+ if (domain->pid < 0 || !domain->overlap) {
gfs_domain_cell_traverse (domain, order, flags, max_depth, func, data);
gfs_domain_homogeneous_bc (domain, flags, max_depth, ov, v);
}
@@ -946,6 +954,85 @@ void gfs_traverse_and_homogeneous_bc (GfsDomain * domain,
gfs_domain_cell_traverse (domain, order, flags, max_depth,
(FttCellTraverseFunc) update_other_cell, &d);
/* Apply homogeneous BC on other boundaries */
+ gts_container_foreach (GTS_CONTAINER (domain),
+ (GtsFunc) update_other_homogeneous_boundaries, &d.b);
+ /* Receive and synchronize */
+ gts_container_foreach (GTS_CONTAINER (domain), (GtsFunc) box_receive_bc, &d.b);
+ gts_container_foreach (GTS_CONTAINER (domain), (GtsFunc) box_synchronize, &d.b.c);
+ }
+}
+
+static void update_other_boundaries (GfsBox * box, BcData * p)
+{
+ FttDirection d;
+ for (d = 0; d < FTT_NEIGHBORS; d++)
+ if (GFS_IS_BOUNDARY (box->neighbor[d]) &&
+ !GFS_IS_BOUNDARY_MPI (box->neighbor[d])) {
+ GfsBoundary * b = GFS_BOUNDARY (box->neighbor[d]);
+ GfsBc * bc = gfs_boundary_lookup_bc (b, p->v);
+
+ if (bc) {
+ b->v = p->v1;
+ bc->v = p->v1;
+ b->type = GFS_BOUNDARY_CENTER_VARIABLE;
+ ftt_face_traverse_boundary (b->root, b->d,
+ FTT_PRE_ORDER, p->flags, p->max_depth,
+ bc->bc, bc);
+ bc->v = p->v;
+ gfs_boundary_send (b);
+ }
+ }
+}
+
+/**
+ * gfs_traverse_and_bc:
+ * @domain: a #GfsDomain.
+ * @order: the order in which the cells are visited - %FTT_PRE_ORDER,
+ * %FTT_POST_ORDER.
+ * @flags: which types of children are to be visited.
+ * @max_depth: the maximum depth of the traversal. Cells below this
+ * depth will not be traversed. If @max_depth is -1 all cells in the
+ * tree are visited.
+ * @func: the function to call for each visited #FttCell.
+ * @data: user data to pass to @func.
+ * @v: a #GfsVariable.
+ * @v1: another #GfsVariable.
+ *
+ * For serial runs, this is identical to calling:
+ *
+ * gfs_domain_cell_traverse (domain, order, flags, max_depth, func, data);
+ * gfs_domain_copy_bc (domain, flags, max_depth, v, v1);
+ *
+ * For parallel runs, the communications needed to apply the boundary
+ * conditions are overlapped with the calls to @func in the bulk of
+ * the domain.
+ */
+void gfs_traverse_and_bc (GfsDomain * domain,
+ FttTraverseType order,
+ FttTraverseFlags flags,
+ gint max_depth,
+ FttCellTraverseFunc func,
+ gpointer data,
+ GfsVariable * v,
+ GfsVariable * v1)
+{
+ g_return_if_fail (domain != NULL);
+
+ if (domain->pid < 0 || !domain->overlap) {
+ gfs_domain_cell_traverse (domain, order, flags, max_depth, func, data);
+ gfs_domain_copy_bc (domain, flags, max_depth, v, v1);
+ }
+ else {
+ TraverseBcData d = {
+ { func, data, order, flags, max_depth },
+ { flags, max_depth, v, v1, FTT_XYZ }
+ };
+ /* Update and send MPI boundary values */
+ gts_container_foreach (GTS_CONTAINER (domain), (GtsFunc) update_mpi_boundaries, &d);
+ /* Update bulk of domain and other boundaries */
+ gfs_domain_cell_traverse (domain, order, flags, max_depth,
+ (FttCellTraverseFunc) update_other_cell, &d);
+ /* Apply BC on other boundaries */
gts_container_foreach (GTS_CONTAINER (domain), (GtsFunc) update_other_boundaries, &d.b);
/* Receive and synchronize */
gts_container_foreach (GTS_CONTAINER (domain), (GtsFunc) box_receive_bc, &d.b);
diff --git a/src/domain.h b/src/domain.h
index 5591734..7b80e0d 100644
--- a/src/domain.h
+++ b/src/domain.h
@@ -71,6 +71,8 @@ struct _GfsDomain {
gint version;
gpointer array;
+
+ gboolean overlap; /* whether to overlap MPI communications with computation */
};
struct _GfsDomainClass {
@@ -172,6 +174,14 @@ void gfs_traverse_and_homogeneous_bc (GfsDomain * domain,
gpointer data,
GfsVariable * ov,
GfsVariable * v);
+void gfs_traverse_and_bc (GfsDomain * domain,
+ FttTraverseType order,
+ FttTraverseFlags flags,
+ gint max_depth,
+ FttCellTraverseFunc func,
+ gpointer data,
+ GfsVariable * v,
+ GfsVariable * v1);
void gfs_domain_face_bc (GfsDomain * domain,
FttComponent c,
GfsVariable * v);
diff --git a/src/event.c b/src/event.c
index 9f74a2c..1b18030 100644
--- a/src/event.c
+++ b/src/event.c
@@ -624,9 +624,9 @@ static gboolean gfs_init_event (GfsEvent * event, GfsSimulation * sim)
while (i) {
VarFunc * vf = i->data;
- gfs_domain_cell_traverse (GFS_DOMAIN (sim), FTT_PRE_ORDER, FTT_TRAVERSE_LEAFS, -1,
- (FttCellTraverseFunc) init_vf, vf);
- gfs_domain_bc (GFS_DOMAIN (sim), FTT_TRAVERSE_LEAFS, -1, vf->v);
+ gfs_traverse_and_bc (GFS_DOMAIN (sim), FTT_PRE_ORDER, FTT_TRAVERSE_LEAFS, -1,
+ (FttCellTraverseFunc) init_vf, vf,
+ vf->v, vf->v);
i = i->next;
}
return TRUE;
@@ -1983,8 +1983,8 @@ static void filter (FttCell * cell, GfsEventFilter * f)
static void filtered (FttCell * cell, GfsEventFilter * f)
{
gdouble dt = gfs_object_simulation (f)->advection_params.dt/f->scale;
- GFS_VARIABLE (cell, f->v->i) = ((1. - dt)*GFS_VARIABLE (cell, f->v->i) +
- dt*GFS_VARIABLE (cell, f->tmp->i));
+ GFS_VALUE (cell, f->v) = ((1. - dt)*GFS_VALUE (cell, f->v) +
+ dt*GFS_VALUE (cell, f->tmp));
}
static gboolean gfs_event_filter_event (GfsEvent * event, GfsSimulation * sim)
@@ -1996,10 +1996,10 @@ static gboolean gfs_event_filter_event (GfsEvent * event, GfsSimulation * sim)
f->tmp = gfs_temporary_variable (GFS_DOMAIN (sim));
gfs_domain_cell_traverse (GFS_DOMAIN (sim), FTT_PRE_ORDER, FTT_TRAVERSE_LEAFS, -1,
(FttCellTraverseFunc) filter, f);
- gfs_domain_cell_traverse (GFS_DOMAIN (sim), FTT_PRE_ORDER, FTT_TRAVERSE_LEAFS, -1,
- (FttCellTraverseFunc) filtered, f);
+ gfs_traverse_and_bc (GFS_DOMAIN (sim), FTT_PRE_ORDER, FTT_TRAVERSE_LEAFS, -1,
+ (FttCellTraverseFunc) filtered, f,
+ f->v, f->v);
gts_object_destroy (GTS_OBJECT (f->tmp));
- gfs_domain_bc (GFS_DOMAIN (sim), FTT_TRAVERSE_LEAFS, -1, f->v);
return TRUE;
}
return FALSE;
diff --git a/src/levelset.c b/src/levelset.c
index 4fce173..188b72e 100644
--- a/src/levelset.c
+++ b/src/levelset.c
@@ -157,15 +157,16 @@ static void variable_distance_event_half (GfsEvent * event, GfsSimulation * sim)
gfs_domain_cell_traverse (domain, FTT_POST_ORDER, FTT_TRAVERSE_NON_LEAFS, -1,
(FttCellTraverseFunc) v->v->fine_coarse, v->v);
- gfs_domain_cell_traverse (domain, FTT_PRE_ORDER, FTT_TRAVERSE_LEAFS, -1,
- (FttCellTraverseFunc) distance_for_stencil, data);
+ gfs_traverse_and_bc (domain, FTT_PRE_ORDER, FTT_TRAVERSE_LEAFS, -1,
+ (FttCellTraverseFunc) distance_for_stencil, data,
+ GFS_VARIABLE1 (event),GFS_VARIABLE1 (event));
gts_object_destroy (data[1]);
gts_object_destroy (data[2]);
}
else
- gfs_domain_cell_traverse (domain, FTT_PRE_ORDER, FTT_TRAVERSE_LEAFS, -1,
- (FttCellTraverseFunc) distance, v);
- gfs_domain_bc (domain, FTT_TRAVERSE_LEAFS, -1, GFS_VARIABLE1 (event));
+ gfs_traverse_and_bc (domain, FTT_PRE_ORDER, FTT_TRAVERSE_LEAFS, -1,
+ (FttCellTraverseFunc) distance, v,
+ GFS_VARIABLE1 (event),GFS_VARIABLE1 (event));
gfs_domain_timer_stop (domain, "distance");
}
diff --git a/src/poisson.c b/src/poisson.c
index 608dae9..0115a21 100644
--- a/src/poisson.c
+++ b/src/poisson.c
@@ -562,7 +562,7 @@ static void correct (FttCell * cell, gpointer * data)
{
GfsVariable * u = data[0];
GfsVariable * dp = data[1];
- GFS_VARIABLE (cell, u->i) += GFS_VARIABLE (cell, dp->i);
+ GFS_VALUE (cell, u) += GFS_VALUE (cell, dp);
}
static void get_from_above (FttCell * parent, GfsVariable * v)
@@ -748,9 +748,9 @@ void gfs_poisson_cycle (GfsDomain * domain,
/* correct on leaf cells */
data[0] = u;
data[1] = dp;
- gfs_domain_cell_traverse (domain, FTT_PRE_ORDER, FTT_TRAVERSE_LEAFS, -1,
- (FttCellTraverseFunc) correct, data);
- gfs_domain_bc (domain, FTT_TRAVERSE_LEAFS, -1, u);
+ gfs_traverse_and_bc (domain, FTT_PRE_ORDER, FTT_TRAVERSE_LEAFS, -1,
+ (FttCellTraverseFunc) correct, data,
+ u, u);
/* compute new residual on leaf cells */
gfs_residual (domain, p->dimension, FTT_TRAVERSE_LEAFS, -1, u, rhs, dia, res);
@@ -1146,9 +1146,9 @@ void gfs_diffusion_cycle (GfsDomain * domain,
/* correct on leaf cells */
data[0] = u;
data[1] = dp;
- gfs_domain_cell_traverse (domain, FTT_PRE_ORDER, FTT_TRAVERSE_LEAFS, -1,
- (FttCellTraverseFunc) correct, data);
- gfs_domain_bc (domain, FTT_TRAVERSE_LEAFS, -1, u);
+ gfs_traverse_and_bc (domain, FTT_PRE_ORDER, FTT_TRAVERSE_LEAFS, -1,
+ (FttCellTraverseFunc) correct, data,
+ u, u);
/* compute new residual on leaf cells */
gfs_diffusion_residual (domain, u, rhs, rhoc, axi, res);
diff --git a/src/source.c b/src/source.c
index 637faa3..1249917 100644
--- a/src/source.c
+++ b/src/source.c
@@ -948,8 +948,9 @@ static void source_viscosity_transverse_flux (GfsSourceGeneric * s,
p.sv = sv;
p.dt = dt;
p.tv = gfs_temporary_variable (domain);
- gfs_domain_traverse_leaves (domain, (FttCellTraverseFunc) compute_transverse, &p);
- gfs_domain_bc (domain, FTT_TRAVERSE_LEAFS, -1, p.tv);
+ gfs_traverse_and_bc (domain, FTT_PRE_ORDER, FTT_TRAVERSE_LEAFS, -1,
+ (FttCellTraverseFunc) compute_transverse, &p,
+ p.tv, p.tv);
gfs_domain_traverse_leaves (domain, (FttCellTraverseFunc) add_viscosity_transverse_flux, &p);
gts_object_destroy (GTS_OBJECT (p.tv));
}
@@ -1163,8 +1164,9 @@ static void source_viscosity_explicit_flux (GfsSourceGeneric * s,
p.sv = sv;
p.dt = dt;
p.tv = gfs_temporary_variable (domain);
- gfs_domain_traverse_leaves (domain, (FttCellTraverseFunc) compute_transverse, &p);
- gfs_domain_bc (domain, FTT_TRAVERSE_LEAFS, -1, p.tv);
+ gfs_traverse_and_bc (domain, FTT_PRE_ORDER, FTT_TRAVERSE_LEAFS, -1,
+ (FttCellTraverseFunc) compute_transverse, &p,
+ p.tv, p.tv);
gfs_domain_traverse_leaves (domain, (FttCellTraverseFunc) add_viscosity_explicit_flux, &p);
if (GFS_IS_AXI (domain) && v->component == FTT_Y)
gfs_domain_traverse_leaves (domain, (FttCellTraverseFunc) add_axisymmetric_term, &p);
diff --git a/src/timestep.c b/src/timestep.c
index 534ac36..df568c8 100644
--- a/src/timestep.c
+++ b/src/timestep.c
@@ -338,12 +338,13 @@ static void mac_projection (GfsDomain * domain,
while (par->niter < par->nitermin ||
(par->residual.infty > par->tolerance && par->niter < par->nitermax)) {
#if 0
- fprintf (stderr, "%d bias: %g first: %g second: %g infty: %g\n",
- par->niter,
- par->residual.bias,
- par->residual.first,
- par->residual.second,
- par->residual.infty);
+ if (domain->pid <= 0)
+ fprintf (stderr, "%d bias: %g first: %g second: %g infty: %g\n",
+ par->niter,
+ par->residual.bias,
+ par->residual.first,
+ par->residual.second,
+ par->residual.infty);
#endif
gfs_poisson_cycle (domain, par, p, div, dia, res1);
par->residual = gfs_domain_norm_residual (domain, FTT_TRAVERSE_LEAFS, -1, apar->dt, res1);
--
Gerris Flow Solver
More information about the debian-science-commits
mailing list