[SCM] Gerris Flow Solver branch, upstream, updated. e8f73a07832050124d2b8bf6c6f35b33180e65a8
Stephane Popinet
popinet at users.sf.net
Tue Nov 24 12:24:17 UTC 2009
The following commit has been merged in the upstream branch:
commit 7bf84874446152a3a15db26154de79cef97d6b44
Author: Stephane Popinet <popinet at users.sf.net>
Date: Sun Jun 28 18:50:08 2009 +1000
New functions gfs_send_boxes() and gfs_receive_boxes()
To move GfsBoxes between processors in parallel.
darcs-hash:20090628085008-d4795-95ff8f66a72ee1f90954ff9f47a38777f7b70ca8.gz
diff --git a/src/boundary.c b/src/boundary.c
index 20d308c..e9d0167 100644
--- a/src/boundary.c
+++ b/src/boundary.c
@@ -1426,8 +1426,9 @@ GfsGEdgeClass * gfs_gedge_class (void)
* @edge: a #GfsGEdge.
*
* Links the two boxes connected by @edge. The boxes are set as their
- * respective neighbors in the direction defined by @edge and their
- * relative positions are set accordingly.
+ * respective neighbors in the direction defined by @edge (note that
+ * their relative positions are not set, call
+ * gfs_box_set_relative_pos() if necessary).
*/
void gfs_gedge_link_boxes (GfsGEdge * edge)
{
@@ -1458,7 +1459,6 @@ void gfs_gedge_link_boxes (GfsGEdge * edge)
(FttCellInitFunc) gfs_cell_init, gfs_box_domain (b1));
b1->neighbor[edge->d] = GTS_OBJECT (b2);
b2->neighbor[FTT_OPPOSITE_DIRECTION (edge->d)] = GTS_OBJECT (b1);
- gfs_box_set_relative_pos (b2, b1, edge->d);
}
}
@@ -1484,7 +1484,7 @@ GfsGEdge * gfs_gedge_new (GfsGEdgeClass * klass,
g_return_val_if_fail (d >= 0 && d < FTT_NEIGHBORS, NULL);
edge = GFS_GEDGE (gts_gedge_new (GTS_GEDGE_CLASS (klass),
- GTS_GNODE (b1), GTS_GNODE (b2)));
+ GTS_GNODE (b1), GTS_GNODE (b2)));
edge->d = d;
gfs_gedge_link_boxes (edge);
@@ -1525,14 +1525,16 @@ static void box_size (FttCell * cell, guint * size)
static void gfs_box_write (GtsObject * object, FILE * fp)
{
GfsBox * box = GFS_BOX (object);
+ FttVector pos;
FttDirection d;
guint size = 0;
GfsDomain * domain = gfs_box_domain (box);
ftt_cell_traverse (box->root, FTT_PRE_ORDER, FTT_TRAVERSE_LEAFS, -1,
(FttCellTraverseFunc) box_size, &size);
- fprintf (fp, "%s { id = %u pid = %d size = %u",
- object->klass->info.name, box->id, box->pid, size);
+ ftt_cell_pos (box->root, &pos);
+ fprintf (fp, "%s { id = %u pid = %d size = %u x = %g y = %g z = %g",
+ object->klass->info.name, box->id, box->pid, size, pos.x, pos.y, pos.z);
for (d = 0; d < FTT_NEIGHBORS; d++)
if (GFS_IS_BOUNDARY (box->neighbor[d])) {
fprintf (fp, " %s = %s",
@@ -1559,27 +1561,35 @@ static void gfs_box_read (GtsObject ** o, GtsFile * fp)
GfsBox * b = GFS_BOX (*o);
GtsObjectClass * klass;
gboolean class_changed = FALSE;
+ FttVector pos = {0., 0., 0.};
GtsFileVariable var[] = {
- {GTS_UINT, "id", TRUE},
- {GTS_INT, "pid", TRUE},
- {GTS_UINT, "size", TRUE},
- {GTS_FILE, "right", TRUE},
- {GTS_FILE, "left", TRUE},
- {GTS_FILE, "top", TRUE},
- {GTS_FILE, "bottom", TRUE},
+ {GTS_UINT, "id", TRUE},
+ {GTS_INT, "pid", TRUE},
+ {GTS_UINT, "size", TRUE},
+ {GTS_DOUBLE, "x", TRUE},
+ {GTS_DOUBLE, "y", TRUE},
+ {GTS_DOUBLE, "z", TRUE},
+ {GTS_FILE, "right", TRUE},
+ {GTS_FILE, "left", TRUE},
+ {GTS_FILE, "top", TRUE},
+ {GTS_FILE, "bottom", TRUE},
#if (!FTT_2D)
- {GTS_FILE, "front", TRUE},
- {GTS_FILE, "back", TRUE},
+ {GTS_FILE, "front", TRUE},
+ {GTS_FILE, "back", TRUE},
#endif /* 3D */
{GTS_NONE}
};
GtsFileVariable * v;
gfloat weight;
- GfsDomain * domain;
+ GfsDomain * domain = GTS_OBJECT (*o)->reserved;
- g_assert (GTS_SLIST_CONTAINEE (b)->containers &&
- !GTS_SLIST_CONTAINEE (b)->containers->next);
- domain = GFS_DOMAIN (GTS_SLIST_CONTAINEE (b)->containers->data);
+ if (domain == NULL) {
+ g_assert (GTS_SLIST_CONTAINEE (b)->containers &&
+ !GTS_SLIST_CONTAINEE (b)->containers->next);
+ domain = GFS_DOMAIN (GTS_SLIST_CONTAINEE (b)->containers->data);
+ }
+ else
+ gts_container_add (GTS_CONTAINER (domain), GTS_CONTAINEE (b));
if (fp->type != GTS_STRING) {
gts_file_error (fp, "expecting a string (GfsBoxClass)");
@@ -1610,9 +1620,13 @@ static void gfs_box_read (GtsObject ** o, GtsFile * fp)
var[0].data = &b->id;
var[1].data = &b->pid;
var[2].data = &b->size;
+ var[3].data = &pos.x;
+ var[4].data = &pos.y;
+ var[5].data = &pos.z;
gts_file_assign_start (fp, var);
if (fp->type == GTS_ERROR)
return;
+
while ((v = gts_file_assign_next (fp, var)))
if (v->type == GTS_FILE) {
GtsObjectClass * boundary_class = gfs_object_class_from_name (fp->token->str);
@@ -1679,6 +1693,21 @@ static void gfs_box_read (GtsObject ** o, GtsFile * fp)
if (GFS_IS_BOUNDARY (b->neighbor[d]))
ftt_cell_set_level (GFS_BOUNDARY (b->neighbor[d])->root, domain->rootlevel);
}
+
+ if (var[3].set || var[4].set || var[5].set) {
+ gdouble size = ftt_cell_size (b->root);
+ FttDirection d;
+ ftt_cell_set_pos (b->root, &pos);
+ for (d = 0; d < FTT_NEIGHBORS; d++)
+ if (GFS_IS_BOUNDARY (b->neighbor[d])) {
+ FttVector bpos = pos;
+ bpos.x += rpos[d].x*size;
+ bpos.y += rpos[d].y*size;
+ bpos.z += rpos[d].z*size;
+ ftt_cell_set_pos (GFS_BOUNDARY (b->neighbor[d])->root, &bpos);
+ }
+ }
+
/* updates weight of domain */
GTS_WGRAPH (domain)->weight += gts_gnode_weight (GTS_GNODE (b)) - weight;
diff --git a/src/domain.c b/src/domain.c
index d1e6475..4deb65e 100644
--- a/src/domain.c
+++ b/src/domain.c
@@ -21,11 +21,14 @@
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
#include "domain.h"
#include "advection.h"
#include "source.h"
#include "solid.h"
+#include "adaptive.h"
#include "config.h"
#ifdef HAVE_MPI
@@ -201,6 +204,7 @@ static void mpi_links (GfsBox * box, GfsDomain * domain)
FTT_OPPOSITE_DIRECTION (d),
pid, id);
}
+#endif /* HAVE_MPI */
static void add_id (GfsBox * box, GPtrArray * ids)
{
@@ -209,46 +213,62 @@ static void add_id (GfsBox * box, GPtrArray * ids)
g_ptr_array_index (ids, box->id - 1) = box;
}
+static GPtrArray * box_ids (GfsDomain * domain)
+{
+ GPtrArray * ids = g_ptr_array_new ();
+ gts_container_foreach (GTS_CONTAINER (domain), (GtsFunc) add_id, ids);
+ return ids;
+}
+
+
static void convert_boundary_mpi_into_edges (GfsBox * box, GPtrArray * ids)
{
+#ifdef HAVE_MPI
+ gint pid = gfs_box_domain (box)->pid;
FttDirection d;
for (d = 0; d < FTT_NEIGHBORS; d++)
if (GFS_IS_BOUNDARY_MPI (box->neighbor[d])) {
GfsBoundaryMpi * b = GFS_BOUNDARY_MPI (box->neighbor[d]);
GfsBox * nbox;
- if (b->id >= 0 && (nbox = g_ptr_array_index (ids, b->id - 1))) {
+ if ((pid < 0 || b->process == pid) /* This is an "internal" boundary */ &&
+ b->id >= 0 && (nbox = g_ptr_array_index (ids, b->id - 1))) {
+ FttDirection od = FTT_OPPOSITE_DIRECTION (d);
if (nbox->pid != b->process)
g_warning ("nbox->pid != b->process");
- else if (!GFS_IS_BOUNDARY_MPI (nbox->neighbor[FTT_OPPOSITE_DIRECTION (d)]))
+ else if (!GFS_IS_BOUNDARY_MPI (nbox->neighbor[od]))
g_warning ("!GFS_IS_BOUNDARY_MPI (nbox->neighbor[FTT_OPPOSITE_DIRECTION (d)])");
else {
- GfsBoundaryMpi * nb = GFS_BOUNDARY_MPI (nbox->neighbor[FTT_OPPOSITE_DIRECTION (d)]);
+ GfsBoundaryMpi * nb = GFS_BOUNDARY_MPI (nbox->neighbor[od]);
if (box->pid != nb->process || box->id != nb->id)
g_warning ("box->pid != nb->process || box->id != nb->id");
else {
gts_object_destroy (GTS_OBJECT (b));
gts_object_destroy (GTS_OBJECT (nb));
- gfs_gedge_new (gfs_gedge_class (), box, nbox, d);
+ gfs_gedge_new (gfs_gedge_class (), nbox, box, od);
}
}
}
}
+ if (pid >= 0)
+ box->pid = pid;
+#else /* not HAVE_MPI */
+ g_assert_not_reached ();
+#endif /* not HAVE_MPI */
}
-#endif /* HAVE_MPI */
static void domain_post_read (GfsDomain * domain, GtsFile * fp)
{
gts_graph_foreach_edge (GTS_GRAPH (domain), (GtsFunc) gfs_gedge_link_boxes, NULL);
- gts_container_foreach (GTS_CONTAINER (domain), (GtsFunc) set_ref_pos, &domain->refpos);
-#ifdef HAVE_MPI
if (domain->pid >= 0) { /* Multiple PEs */
+#ifdef HAVE_MPI
GSList * removed = NULL;
guint np = 0;
gpointer data[3];
int comm_size;
+ gts_container_foreach (GTS_CONTAINER (domain), (GtsFunc) set_ref_pos, &domain->refpos);
data[0] = domain;
data[1] = &removed;
data[2] = &np;
@@ -261,18 +281,21 @@ static void domain_post_read (GfsDomain * domain, GtsFile * fp)
}
g_slist_foreach (removed, (GFunc) mpi_links, domain);
g_slist_free (removed);
+#else /* not HAVE_MPI */
+ g_assert_not_reached ();
+#endif /* not HAVE_MPI */
}
else { /* Single PE */
/* Create array for fast linking of ids to GfsBox pointers */
- GPtrArray * ids = g_ptr_array_new ();
- gts_container_foreach (GTS_CONTAINER (domain), (GtsFunc) add_id, ids);
+ GPtrArray * ids = box_ids (domain);
/* Convert GfsBoundaryMpi into graph edges */
gts_container_foreach (GTS_CONTAINER (domain), (GtsFunc) convert_boundary_mpi_into_edges, ids);
g_ptr_array_free (ids, TRUE);
+
+ gts_container_foreach (GTS_CONTAINER (domain), (GtsFunc) set_ref_pos, &domain->refpos);
}
-#endif /* HAVE_MPI */
gfs_domain_match (domain);
}
@@ -2571,6 +2594,7 @@ void gfs_cell_write_binary (const FttCell * cell, FILE * fp,
fwrite (s->solid->s, sizeof (gdouble), FTT_NEIGHBORS, fp);
fwrite (&s->solid->a, sizeof (gdouble), 1, fp);
fwrite (&s->solid->cm.x, sizeof (gdouble), FTT_DIMENSION, fp);
+ fwrite (&s->solid->ca.x, sizeof (gdouble), FTT_DIMENSION, fp);
}
else {
gdouble a = -1.;
@@ -2619,7 +2643,8 @@ void gfs_cell_read_binary (FttCell * cell, GtsFile * fp, GfsDomain * domain)
s->solid = g_malloc0 (sizeof (GfsSolidVector));
s->solid->s[0] = s0;
- if (gts_file_read (fp, &s->solid->s[1], sizeof (gdouble), FTT_NEIGHBORS - 1) != FTT_NEIGHBORS - 1) {
+ if (gts_file_read (fp, &s->solid->s[1], sizeof (gdouble), FTT_NEIGHBORS - 1)
+ != FTT_NEIGHBORS - 1) {
gts_file_error (fp, "expecting numbers (solid->s[1..%d])", FTT_NEIGHBORS - 1);
return;
}
@@ -2631,6 +2656,10 @@ void gfs_cell_read_binary (FttCell * cell, GtsFile * fp, GfsDomain * domain)
gts_file_error (fp, "expecting numbers (solid->cm[0..%d])", FTT_DIMENSION - 1);
return;
}
+ if (gts_file_read (fp, &s->solid->ca.x, sizeof (gdouble), FTT_DIMENSION) != FTT_DIMENSION) {
+ gts_file_error (fp, "expecting numbers (solid->ca[0..%d])", FTT_DIMENSION - 1);
+ return;
+ }
}
i = domain->variables_io;
@@ -3631,3 +3660,158 @@ void gfs_domain_filter (GfsDomain * domain, GfsVariable * v, GfsVariable * fv)
else
gfs_domain_copy_bc (domain, FTT_TRAVERSE_LEAFS, -1, v, fv);
}
+
+/**
+ * gfs_send_objects:
+ * @list: a list of #GtsObject.
+ * @dest: the rank of the destination PE.
+ *
+ * Sends the objects in @list to PE @dest of a parallel simulation.
+ *
+ * Note that this functions assumes that the write() method of the
+ * #GtsObject sent begins by writing the object class name.
+ */
+void gfs_send_objects (GSList * list, int dest)
+{
+#ifdef HAVE_MPI
+ FILE * fp = tmpfile ();
+ int fd = fileno (fp);
+ struct stat sb;
+ while (list) {
+ GtsObject * object = list->data;
+ g_assert (object->klass->write != NULL);
+ (* object->klass->write) (object, fp);
+ fputc ('\n', fp);
+ list = list->next;
+ }
+ fflush (fp);
+ g_assert (fstat (fd, &sb) != -1);
+ long length = sb.st_size;
+ MPI_Send (&length, 1, MPI_LONG, dest, 0, MPI_COMM_WORLD);
+ /* g_log (G_LOG_DOMAIN, G_LOG_LEVEL_MESSAGE, "sending %ld bytes to PE %d", length, dest); */
+ if (length > 0) {
+ char * buf = mmap (NULL, length, PROT_READ, MAP_PRIVATE, fd, 0);
+ g_assert (buf != MAP_FAILED);
+ MPI_Send (buf, length, MPI_BYTE, dest, 1, MPI_COMM_WORLD);
+ munmap (buf, length);
+ }
+ fclose (fp);
+#endif /* HAVE_MPI */
+}
+
+/**
+ * gfs_receive_objects:
+ * @domain: a #GfsDomain.
+ * @src: the rank of the source PE.
+ *
+ * Receives a list of #GtsObject from PE @src of a parallel simulation.
+ *
+ * Returns: a list of newly-allocated objects.
+ */
+GSList * gfs_receive_objects (GfsDomain * domain, int src)
+{
+ g_return_val_if_fail (domain != NULL, NULL);
+
+#ifdef HAVE_MPI
+ MPI_Status status;
+ long length;
+ MPI_Recv (&length, 1, MPI_LONG, src, 0, MPI_COMM_WORLD, &status);
+ /* g_log (G_LOG_DOMAIN, G_LOG_LEVEL_MESSAGE, "receiving %ld bytes from PE %d", length, src); */
+ if (length > 0) {
+ char * buf = g_malloc (length);
+ MPI_Recv (buf, length, MPI_BYTE, src, 1, MPI_COMM_WORLD, &status);
+ FILE * f = tmpfile ();
+ fwrite (buf, 1, length, f);
+ rewind (f);
+ GtsFile * fp = gts_file_new (f);
+ GSList * list = NULL;
+ while (fp->type == GTS_STRING) {
+ GtsObjectClass * klass = gfs_object_class_from_name (fp->token->str);
+ if (klass == NULL)
+ g_error ("gfs_receive_object():%d:%d: unknown class '%s'",
+ fp->line, fp->pos, fp->token->str);
+ GtsObject * object = gts_object_new (klass);
+ gfs_object_simulation_set (object, domain);
+ g_assert (klass->read);
+ (* klass->read) (&object, fp);
+ if (fp->type == GTS_ERROR)
+ g_error ("gfs_receive_object():%d:%d: %s", fp->line, fp->pos, fp->error);
+ list = g_slist_prepend (list, object);
+ }
+ gts_file_destroy (fp);
+ fclose (f);
+ g_free (buf);
+ return list;
+ }
+#endif /* HAVE_MPI */
+ return NULL;
+}
+
+static void unlink_box (GfsBox * box, gint * dest)
+{
+#ifdef HAVE_MPI
+ FttDirection d;
+ for (d = 0; d < FTT_NEIGHBORS; d++)
+ if (GFS_IS_BOX (box->neighbor[d])) {
+ GfsBox * nbox = GFS_BOX (box->neighbor[d]);
+ FttDirection od = FTT_OPPOSITE_DIRECTION (d);
+ nbox->neighbor[od] = NULL;
+ gfs_boundary_mpi_new (gfs_boundary_mpi_class (), nbox, od, *dest, box->id);
+ box->neighbor[d] = NULL;
+ gfs_boundary_mpi_new (gfs_boundary_mpi_class (), box, d, nbox->pid, nbox->id);
+ }
+#else /* doesn't HAVE_MPI */
+ g_assert_not_reached ();
+#endif
+}
+
+static void setup_binary_IO (GfsDomain * domain)
+{
+ /* make sure that all the variables are sent */
+ g_slist_free (domain->variables_io);
+ domain->variables_io = NULL;
+ GSList * i = domain->variables;
+ while (i) {
+ if (GFS_VARIABLE1 (i->data)->name)
+ domain->variables_io = g_slist_append (domain->variables_io, i->data);
+ i = i->next;
+ }
+ domain->binary = TRUE;
+}
+
+void gfs_send_boxes (GfsDomain * domain, GSList * boxes, int dest)
+{
+ g_return_if_fail (domain != NULL);
+ g_return_if_fail (dest != domain->pid);
+
+ g_slist_foreach (boxes, (GFunc) unlink_box, &dest);
+ setup_binary_IO (domain);
+ gfs_send_objects (boxes, dest);
+ g_slist_foreach (boxes, (GFunc) gts_object_destroy, NULL);
+ if (boxes)
+ gfs_domain_reshape (domain, gfs_domain_depth (domain));
+}
+
+/**
+ * gfs_receive_boxes:
+ * @domain:
+ */
+GSList * gfs_receive_boxes (GfsDomain * domain, int src)
+{
+ g_return_val_if_fail (domain != NULL, NULL);
+ g_return_val_if_fail (src != domain->pid, NULL);
+
+ setup_binary_IO (domain);
+ GSList * boxes = gfs_receive_objects (domain, src);
+ if (boxes) {
+ /* Create array for fast linking of ids to GfsBox pointers */
+ GPtrArray * ids = box_ids (domain);
+
+ /* Convert internal GfsBoundaryMpi into graph edges */
+ g_slist_foreach (boxes, (GFunc) convert_boundary_mpi_into_edges, ids);
+ g_ptr_array_free (ids, TRUE);
+
+ gfs_domain_reshape (domain, gfs_domain_depth (domain));
+ }
+ return boxes;
+}
diff --git a/src/domain.h b/src/domain.h
index 9bfa1a5..f108a22 100644
--- a/src/domain.h
+++ b/src/domain.h
@@ -296,6 +296,16 @@ void gfs_domain_sum (GfsDomain * domain,
void gfs_domain_filter (GfsDomain * domain,
GfsVariable * v,
GfsVariable * fv);
+void gfs_send_objects (GSList * list,
+ int dest);
+GSList * gfs_receive_objects (GfsDomain * domain,
+ int src);
+void gfs_send_boxes (GfsDomain * domain,
+ GSList * boxes,
+ int dest);
+GSList * gfs_receive_boxes (GfsDomain * domain,
+ int src);
+
/**
* gfs_domain_face_fraction:
* @domain; a #GfsDomain.
diff --git a/src/ftt.c b/src/ftt.c
index 7e4c9e3..0429cbd 100644
--- a/src/ftt.c
+++ b/src/ftt.c
@@ -608,8 +608,6 @@ static void update_neighbor_match (FttCell * cell,
FttCellChildren children;
guint i, n;
- g_assert (oct->neighbors.c[d] == NULL ||
- oct->neighbors.c[d] == neighbor);
oct->neighbors.c[d] = neighbor;
if (ftt_cell_level (neighbor) < oct->level) {
@@ -682,12 +680,10 @@ void ftt_cell_set_neighbor_match (FttCell * root,
g_return_if_fail (ftt_cell_level (root) == ftt_cell_level (neighbor));
- g_return_if_fail (FTT_ROOT_CELL (root)->neighbors.c[d] == NULL);
FTT_ROOT_CELL (root)->neighbors.c[d] = neighbor;
update_neighbor_match (root, d, init, init_data);
od = FTT_OPPOSITE_DIRECTION (d);
- g_return_if_fail (FTT_ROOT_CELL (neighbor)->neighbors.c[od] == NULL);
FTT_ROOT_CELL (neighbor)->neighbors.c[od] = root;
update_neighbor_match (neighbor, od, init, init_data);
}
--
Gerris Flow Solver
More information about the debian-science-commits
mailing list