[Pkg-ofed-commits] [libfabric] 78/123: prov/psm: code refactoring of the AM-based messaging
Ana Beatriz Guerrero López
ana at moszumanska.debian.org
Sat Oct 22 12:28:32 UTC 2016
This is an automated email from the git hooks/post-receive script.
ana pushed a commit to annotated tag v1.1.1
in repository libfabric.
commit ad806f63a3bb276e085f3a37b7f4e20d9da4f17b
Author: Jianxin Xiong <jianxin.xiong at intel.com>
Date: Thu Sep 17 17:14:09 2015 -0700
prov/psm: code refactoring of the AM-based messaging
(1) always use the send queue to defer the processing of ready-to-send
request to outside the AM handler. Remove the busy polling from the
send path.
(2) remove the poorly performed progress thread and purely rely on
manual progress.
(3) remove the now redundant state tracking of the send request.
Signed-off-by: Jianxin Xiong <jianxin.xiong at intel.com>
---
prov/psm/src/psmx.h | 20 ----------------
prov/psm/src/psmx_am.c | 60 ++++++------------------------------------------
prov/psm/src/psmx_msg2.c | 25 +-------------------
prov/psm/src/psmx_rma.c | 1 -
4 files changed, 8 insertions(+), 98 deletions(-)
diff --git a/prov/psm/src/psmx.h b/prov/psm/src/psmx.h
index b2734d2..7c0a5eb 100644
--- a/prov/psm/src/psmx.h
+++ b/prov/psm/src/psmx.h
@@ -125,10 +125,6 @@ union psmx_pi {
#define PSMX_AM_DATA 0x20000000
#define PSMX_AM_FORCE_ACK 0x10000000
-#ifndef PSMX_AM_USE_SEND_QUEUE
-#define PSMX_AM_USE_SEND_QUEUE 0
-#endif
-
enum {
PSMX_AM_REQ_WRITE = 1,
PSMX_AM_REQ_WRITE_LONG,
@@ -146,13 +142,6 @@ enum {
PSMX_AM_REP_ATOMIC_COMPWRITE,
};
-enum {
- PSMX_AM_STATE_NEW,
- PSMX_AM_STATE_QUEUED,
- PSMX_AM_STATE_PROCESSED,
- PSMX_AM_STATE_DONE
-};
-
struct psmx_am_request {
int op;
union {
@@ -203,7 +192,6 @@ struct psmx_am_request {
uint64_t cq_flags;
struct fi_context fi_context;
struct psmx_fid_ep *ep;
- int state;
int no_event;
int error;
struct slist_entry list_entry;
@@ -258,19 +246,11 @@ struct psmx_fid_domain {
int am_initialized;
-#if PSMX_AM_USE_SEND_QUEUE
- pthread_cond_t progress_cond;
- pthread_mutex_t progress_mutex;
- pthread_t progress_thread;
-#endif
-
/* incoming req queue for AM based RMA request. */
struct psmx_req_queue rma_queue;
-#if PSMX_AM_USE_SEND_QUEUE
/* send queue for AM based messages. */
struct psmx_req_queue send_queue;
-#endif
/* recv queue for AM based messages. */
struct psmx_req_queue recv_queue;
diff --git a/prov/psm/src/psmx_am.c b/prov/psm/src/psmx_am.c
index ce462f7..e28c282 100644
--- a/prov/psm/src/psmx_am.c
+++ b/prov/psm/src/psmx_am.c
@@ -49,22 +49,17 @@ int psmx_am_progress(struct psmx_fid_domain *domain)
struct psmx_am_request *req;
struct psmx_trigger *trigger;
-#if PSMX_AM_USE_SEND_QUEUE
- pthread_mutex_lock(&domain->send_queue.lock);
- while (!slist_empty(&domain->send_queue.list)) {
- item = slist_remove_head(&domain->send_queue.list);
- req = container_of(item, struct psmx_am_request, list_entry);
- if (req->state == PSMX_AM_STATE_DONE) {
- free(req);
- }
- else {
+ if (psmx_env.am_msg) {
+ pthread_mutex_lock(&domain->send_queue.lock);
+ while (!slist_empty(&domain->send_queue.list)) {
+ item = slist_remove_head(&domain->send_queue.list);
+ req = container_of(item, struct psmx_am_request, list_entry);
pthread_mutex_unlock(&domain->send_queue.lock);
psmx_am_process_send(domain, req);
pthread_mutex_lock(&domain->send_queue.lock);
}
+ pthread_mutex_unlock(&domain->send_queue.lock);
}
- pthread_mutex_unlock(&domain->send_queue.lock);
-#endif
if (psmx_env.tagged_rma) {
pthread_mutex_lock(&domain->rma_queue.lock);
@@ -91,33 +86,6 @@ int psmx_am_progress(struct psmx_fid_domain *domain)
return 0;
}
-#if PSMX_AM_USE_SEND_QUEUE
-static void *psmx_am_async_progress(void *args)
-{
- struct psmx_fid_domain *domain = args;
- struct timespec timeout;
-
- timeout.tv_sec = 1;
- timeout.tv_nsec = 1000;
-
- pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
- pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
-
- while (1) {
- pthread_mutex_lock(&domain->progress_mutex);
- pthread_cond_wait(&domain->progress_cond, &domain->progress_mutex);
- pthread_mutex_unlock(&domain->progress_mutex);
- pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL);
-
- psmx_am_progress(domain);
-
- pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
- }
-
- return NULL;
-}
-#endif
-
int psmx_am_init(struct psmx_fid_domain *domain)
{
psm_ep_t psm_ep = domain->psm_ep;
@@ -154,32 +122,18 @@ int psmx_am_init(struct psmx_fid_domain *domain)
slist_init(&domain->recv_queue.list);
slist_init(&domain->unexp_queue.list);
slist_init(&domain->trigger_queue.list);
+ slist_init(&domain->send_queue.list);
pthread_mutex_init(&domain->rma_queue.lock, NULL);
pthread_mutex_init(&domain->recv_queue.lock, NULL);
pthread_mutex_init(&domain->unexp_queue.lock, NULL);
pthread_mutex_init(&domain->trigger_queue.lock, NULL);
-#if PSMX_AM_USE_SEND_QUEUE
- slist_init(&domain->send_queue.list);
pthread_mutex_init(&domain->send_queue.lock, NULL);
- pthread_mutex_init(&domain->progress_mutex, NULL);
- pthread_cond_init(&domain->progress_cond, NULL);
- err = pthread_create(&domain->progress_thread, NULL, psmx_am_async_progress, (void *)domain);
-#endif
return err;
}
int psmx_am_fini(struct psmx_fid_domain *domain)
{
-#if PSMX_AM_USE_SEND_QUEUE
- if (domain->progress_thread) {
- pthread_cancel(domain->progress_thread);
- pthread_join(domain->progress_thread, NULL);
- pthread_mutex_destroy(&domain->progress_mutex);
- pthread_cond_destroy(&domain->progress_cond);
- }
-#endif
-
return 0;
}
diff --git a/prov/psm/src/psmx_msg2.c b/prov/psm/src/psmx_msg2.c
index 6df922c..a99d6c6 100644
--- a/prov/psm/src/psmx_msg2.c
+++ b/prov/psm/src/psmx_msg2.c
@@ -32,16 +32,13 @@
#include "psmx.h"
-#if PSMX_AM_USE_SEND_QUEUE
static inline void psmx_am_enqueue_send(struct psmx_fid_domain *domain,
struct psmx_am_request *req)
{
pthread_mutex_lock(&domain->send_queue.lock);
- req->state = PSMX_AM_STATE_QUEUED;
slist_insert_tail(&req->list_entry, &domain->send_queue.list);
pthread_mutex_unlock(&domain->send_queue.lock);
}
-#endif
static inline void psmx_am_enqueue_recv(struct psmx_fid_domain *domain,
struct psmx_am_request *req)
@@ -260,16 +257,10 @@ int psmx_am_msg_handler(psm_am_token_t token, psm_epaddr_t epaddr,
if (args[2].u64) { /* more to send */
req->send.peer_context = (void *)(uintptr_t)args[2].u64;
-#if PSMX_AM_USE_SEND_QUEUE
/* psm_am_request_short() can't be called inside the handler.
* put the request into a queue and process it later.
*/
psmx_am_enqueue_send(req->ep->domain, req);
- if (req->ep->domain->progress_thread)
- pthread_cond_signal(&req->ep->domain->progress_cond);
-#else
- req->send.peer_ready = 1;
-#endif
}
else { /* done */
if (req->ep->send_cq && !req->no_event) {
@@ -292,10 +283,7 @@ int psmx_am_msg_handler(psm_am_token_t token, psm_epaddr_t epaddr,
if (req->ep->send_cntr)
psmx_cntr_inc(req->ep->send_cntr);
- if (req->state == PSMX_AM_STATE_QUEUED)
- req->state = PSMX_AM_STATE_DONE;
- else
- free(req);
+ free(req);
}
break;
@@ -315,8 +303,6 @@ int psmx_am_process_send(struct psmx_fid_domain *domain, struct psmx_am_request
uint64_t offset;
int err;
- req->state = PSMX_AM_STATE_PROCESSED;
-
offset = req->send.len_sent;
len = req->send.len - offset;
@@ -574,15 +560,6 @@ static ssize_t _psmx_send2(struct fid_ep *ep, const void *buf, size_t len,
PSMX_AM_MSG_HANDLER, args, 4,
(void *)buf, msg_size, am_flags, NULL, NULL);
-#if ! PSMX_AM_USE_SEND_QUEUE
- if (len > msg_size) {
- while (!req->send.peer_ready)
- psm_poll(ep_priv->domain->psm_ep);
-
- psmx_am_process_send(ep_priv->domain, req);
- }
-#endif
-
return psmx_errno(err);
}
diff --git a/prov/psm/src/psmx_rma.c b/prov/psm/src/psmx_rma.c
index d9ea7a7..590c664 100644
--- a/prov/psm/src/psmx_rma.c
+++ b/prov/psm/src/psmx_rma.c
@@ -36,7 +36,6 @@ static inline void psmx_am_enqueue_rma(struct psmx_fid_domain *domain,
struct psmx_am_request *req)
{
pthread_mutex_lock(&domain->rma_queue.lock);
- req->state = PSMX_AM_STATE_QUEUED;
slist_insert_tail(&req->list_entry, &domain->rma_queue.list);
pthread_mutex_unlock(&domain->rma_queue.lock);
}
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-ofed/libfabric.git
More information about the Pkg-ofed-commits
mailing list