//------------------------------------------------------------
// $Id: mpi2communicationMPI.cc,v 1.3.2.1 2000/06/14 17:46:38 yamauchi Exp $
//------------------------------------------------------------
//	mpi2 communication 
//	1996/8/15
//	(C) Copyright YAMAUCHI Hitoshi 
// 	通信関係 実装は MPI (commMpi2 関数群 でカプセル化)
// 
// *注意* 現バージョンは endian が同一であることを仮定している
// Notice: this version assume all machine has the same endian.
// 
//--	昨日、井形に佐々木恒、高橋さんに藤君が来たので大ちゃんの家で飲
//--	んだ。楽しかった。 
//--	最近 Evangerion を見だした。結構面白いが、そろそろ庵野さんも世
//--	界中で認められるような作品を作って欲しい。それだけのことができ
//--	るはず。まだわからんが結局力で戦争している感じだ。でも面白ろい
//--	からいいか。まだ数話しか見ていないので続きが楽しみ。
//------------------------------------------------------------
#include <mpi.h>
#include "mpi2.h"
#include <unistd.h>		// for sleep 
// extern "C" void usleep(unsigned int);
#include "Option.h"
#include "mpi2communication.h"
#include "triangleObject.h"
#include "sphereObject.h"
#include "req_handle_t.h"
#include "volume.h"
#include "sysinfo.h"
#include "viewdata.h"
#include "SpaceParam.h"
#include "Intensity.h"
#include "PEStat.h"

//------------------------------------------------------------
// init and end
//------------------------------------------------------------
// MPI の初期化と size, rank の取得
//------------------------------
void commMpi2_Init(int *pArgc, char ***pArgv)
{
    MPI_Init(pArgc, pArgv);
}

void commMpi2_SizeAndRank(int *pSize, int *pRank)
{
    MPI_Comm_size(MPI_COMM_WORLD, pSize);
    MPI_Comm_rank(MPI_COMM_WORLD, pRank);
}

void commMpi2_Finalize(void)
{
    MPI_Finalize();
}

void commMpi2_exit(int err_number)
{
    MPI_Abort(MPI_COMM_WORLD, err_number);
}

void commMpi2_Barrier(void)
{
    MPI_Barrier(MPI_COMM_WORLD);
}

//------------------------------------------------------------
// broadcast 
//------------------------------------------------------------
// sysinfo
//------------------------------
void commMpi2_BroadSysinfo(SysInfoClass    *pSys,
			   ViewDataClass   *pVw,
			   SpaceParam *pSp)
{
    int root = SysStat::getInstance()->getHostRank();
    //     cout << "***** debug :: Broad casting sysinfo .. root rank  = " 
    // 	<< root << endl;
    assert((root >= 0) && (root < SysStat::getInstance()->getCommSize()));
    MPI_Bcast(pSys, sizeof(SysInfoClass),    MPI_BYTE, root, MPI_COMM_WORLD);
    MPI_Bcast(pVw,  sizeof(ViewDataClass),   MPI_BYTE, root, MPI_COMM_WORLD);
    MPI_Bcast(pSp,  sizeof(SpaceParam), MPI_BYTE, root, MPI_COMM_WORLD);
}

//------------------------------
// 空間中に物体が存在するかどうかの表
//------------------------------
// OBJINSPC_IMPLEMENT_TYPE (=char) の一次元配列で送信する。
// 呼び出し側は pDatArray を用意し、利用後解放する
void commMpi2_BroadObjInSpc(int byteSize, 
			    OBJINSPC_IMPLEMENT_TYPE *pDatArray)
{
    int root = SysStat::getInstance()->getHostRank();
    assert((root >= 0) && (root < SysStat::getInstance()->getCommSize()));
    MPI_Bcast(pDatArray, byteSize, MPI_BYTE, root, MPI_COMM_WORLD);
}

//------------------------------
// proximity clouds 法用 距離マップ 
//------------------------------
//tokunaga changed
void commMpi2_BroadPRCSkipMap(int byteSize, 
			      OBJINSPC_IMPLEMENT_TYPE *pDatArray)
{
    int root = SysStat::getInstance()->getHostRank();
    assert((root >= 0) && (root < SysStat::getInstance()->getCommSize()));
    MPI_Bcast(pDatArray, byteSize, MPI_BYTE, root, MPI_COMM_WORLD);
}


//------------------------------------------------------------
// send, recv object
//------------------------------------------------------------
void commMpi2_sendObject(int destRank, objBaseClass *pObj)
{
    MPI_Request	requestID;
    MPI_Status  stat;
    switch(pObj->who())
    {
    case TRIANGLE_OBJ_E:
	MPI_Isend(pObj, sizeof(triangleObjectClass), MPI_BYTE, 
		  destRank, TAG_TRIANGLE_OBJ_e, MPI_COMM_WORLD, &requestID);
	// 	cout << "MM [" << TRIANGLE_OBJ_E << "] tri obj sended to    " 
	// 	    << destRank << endl;
	break;
    case SPHERE_OBJ_E:
	MPI_Isend(pObj, sizeof(sphereObjectClass), MPI_BYTE, 
		  destRank, TAG_SPHERE_OBJ_e, MPI_COMM_WORLD, &requestID);
	// 	cout << "MM [" << SPHERE_OBJ_E   << "] sphere obj sended to " 
	// 	    << destRank << endl;
	break;
    case NULL_OBJ_E:
	MPI_Isend(pObj, sizeof(nullObjectClass), MPI_BYTE, 
		  destRank, TAG_NULL_OBJ_e, MPI_COMM_WORLD, &requestID);
	// 	cout << "MM [" << NULL_OBJ_E     << "] null obj sended to   " 
	// 	    << destRank << endl;
	break;
    default:
	cerr << "Error! : send object has illegal tag. dest = " << 
	    destRank << endl;
	commMpi2_exit(1);
	break;
    }	
    MPI_Wait(&requestID, &stat);
}

// 1 つの object を受け取る
void commMpi2_recvObject(int srcRank, int destRank, objBaseClass **ppObj)
{
    MPI_Request reqID;
    MPI_Status  stat;
    MPI_Status  probeStat;
    int    	isMessage = 0;	// MPI で 0 はまだメッセージを受けていないこと

    // getting and waiting message
    while(isMessage == 0)
    {
        MPI_Iprobe(srcRank, MPI_ANY_TAG, MPI_COMM_WORLD, 
		   &isMessage, &probeStat);
	if(isMessage == 0)
	{
	    // 	    if(Option.getDebugMode() == YES)
	    // 	    {
	    // 		cout << "== rank " << destRank << 
	    // 		    " get no message, usleep(" << RETRY_WAIT_TIME << 
	    // 			") and continue" << endl;
	    // 	    }
            usleep(POLLING_SLEEP_TIME);
        }
    }

    // now gotten message
    switch(probeStat.MPI_TAG)
    {
    case TAG_TRIANGLE_OBJ_e:
	(*ppObj) = new triangleObjectClass;
	MPI_Irecv((*ppObj), sizeof(triangleObjectClass), MPI_BYTE, 
		  srcRank, TAG_TRIANGLE_OBJ_e, MPI_COMM_WORLD, &reqID);
	MPI_Wait(&reqID, &stat);
	// 	cout << "== [" << TRIANGLE_OBJ_E << "] tri obj received from "
	// 	    << srcRank << " to " << destRank << endl;
	break;

    case TAG_SPHERE_OBJ_e:
	(*ppObj) = new sphereObjectClass;
	MPI_Irecv((*ppObj), sizeof(sphereObjectClass), MPI_BYTE, 
		  srcRank, TAG_SPHERE_OBJ_e, MPI_COMM_WORLD, &reqID);
	MPI_Wait(&reqID, &stat);
	// 	cout << "== [" << SPHERE_OBJ_E << "] sphere obj received from "
	// 	    << srcRank << " to " << destRank << endl;
	break;

    case TAG_NULL_OBJ_e:
	(*ppObj) = new nullObjectClass;
	MPI_Irecv((*ppObj), sizeof(nullObjectClass), MPI_BYTE, 
		  srcRank, TAG_NULL_OBJ_e, MPI_COMM_WORLD, &reqID);
	MPI_Wait(&reqID, &stat);
	// 	cout << "== [" << NULL_OBJ_E << "] null obj received from "
	// 	    << srcRank << " to " << destRank << endl;
	break;
    default:
	cerr << "Error! : recv object has illegal tag. (src, dest) = (" 
	    << srcRank << ", " << destRank << ") tag = " << 
		probeStat.MPI_TAG << endl;
	commMpi2_exit(1);
	break;
    }
    // receive object (*ppObj)->print("== recv");
}


//------------------------------------------------------------
// send, recv volume 
// 	volume は固定長の物体ではないため別にしてあるが統一すべきかも
// 	しれない。
//------------------------------------------------------------
// send volume
void commMpi2_sendVolume(int dstRank, VolumeClass *pVolume)
{
    MPI_Request	requestID;
    MPI_Status  stat;

    double *pBuf = pVolume->newSerializedArray();
    const int totalDoubles = pVolume->getSerializedSize();

    MPI_Isend(pBuf, totalDoubles, MPI_DOUBLE, 
 	      dstRank, TAG_VOLUME_OBJ_e, MPI_COMM_WORLD, &requestID);

    //     cout << "MM [" << TAG_VOLUME_OBJ_e << "] volume sended to    " 
    // 	<< dstRank << endl;

    MPI_Wait(&requestID, &stat);

    delete [] pBuf;
}

// recv volume
void commMpi2_recvVolume(int srcRank, int dstRank, VolumeClass **ppVolume)
{
    MPI_Request reqID;
    MPI_Status  stat;
    MPI_Status  probeStat;
    int    	isMessage = 0;	// MPI で 0 はまだメッセージを受けていないこと

    // getting and waiting message
    while(isMessage == 0)
    {
        MPI_Iprobe(srcRank, MPI_ANY_TAG, MPI_COMM_WORLD, 
		   &isMessage, &probeStat);
	if(isMessage == 0)
	{
	    // 	    if(Option.getDebugMode() == YES)
	    // 	    {
	    // 		cout << "== rank " << destRank << 
	    // 		    " get no message, usleep(" << RETRY_WAIT_TIME << 
	    // 			") and continue" << endl;
	    // 	    }
            usleep(POLLING_SLEEP_TIME);
        }
    }

    // now we get a message
    if(probeStat.MPI_TAG != TAG_VOLUME_OBJ_e)
    {
	cerr << "Internal Error! : recv volume has illegal tag. "
	    "(src, dest) = (" << srcRank << ", " << dstRank << 
		") tag = " << probeStat.MPI_TAG << endl;
	commMpi2_exit(1);
    }else
    {
	int totalDoulble = 0;
	MPI_Get_count(&probeStat, MPI_DOUBLE, &totalDoulble);
	assert(totalDoulble > 0);

	double *pBuf = new double [totalDoulble];

	MPI_Irecv(pBuf, totalDoulble, MPI_DOUBLE, 
		  srcRank, TAG_VOLUME_OBJ_e, MPI_COMM_WORLD, &reqID);
	MPI_Wait(&reqID, &stat);

	(*ppVolume) = new VolumeClass;
	(*ppVolume)->deserialize(pBuf);

	delete [] pBuf;

// 	cout << "== [" << TAG_VOLUME_OBJ_e << "] volume received from "
// 	    << srcRank << " to " << dstRank << endl;
    }
}


//------------------------------------------------------------
// send, recv material
//------------------------------------------------------------
void commMpi2_BroadMaterial(materialClass *pMat)
{
    static const int root = SysStat::getInstance()->getHostRank();
    //     cout << "***** debug :: Broad casting material .. root rank  = " 
    // 	<< root << endl;
    assert((root >= 0) && (root < SysStat::getInstance()->getCommSize()));
    MPI_Bcast(pMat, sizeof(materialClass), MPI_BYTE, root, MPI_COMM_WORLD);
}


//------------------------------------------------------------
// recv probe 
// 	pSrcRank どこから受けられるか : 
//		NO が返った場合には、*pSrcRank は無意味である
//	recvStaff なにを受けるか(ray, intensity, object....)
//------------------------------------------------------------
// canRecv ... 受けるものを指定する
//	@param pSrcRank		どこから受けたか
//	@param recvStaff	受けるもので知りたいもののタグ
//	@param pRecvByteSize	受けた大きさ (Bytes)
//------------------------------
YorN commMpi2_canRecv(int *pSrcRank, enum communication_tags_e recvStaff,
		      int *pRecvByteSize)
{
    MPI_Status  probeStat;
    int isMessage;	
    
    isMessage = 0;		// MPI で 0 はまだメッセージを受けていないこと
    MPI_Iprobe(MPI_ANY_SOURCE, recvStaff, MPI_COMM_WORLD, 
	       &isMessage, &probeStat);

    if(isMessage == 0)
    {
	return(NO);
    }else
    {
	*pSrcRank = probeStat.MPI_SOURCE;
	int bytecount;
	MPI_Get_count(&probeStat, MPI_BYTE, &bytecount);
	(*pRecvByteSize) = bytecount;
	return(YES);
    }
}


//------------------------------
// canRecvObject ... 物体を受ける。どの物体を受けたかが返る
//------------------------------
YorN commMpi2_canRecvObject(int *pSrcRank, int *pRecvTag)
{
    static MPI_Status  probeStat;
    static int isMessage;	
    
    isMessage = 0;		// MPI で 0 はまだメッセージを受けていないこと
    MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, 
	       &isMessage, &probeStat);

    if(isMessage == 0)
    {
	return(NO);
    }else
    {
	// 物体かどうか
	if((probeStat.MPI_TAG == TAG_TRIANGLE_OBJ_e) || 
	   (probeStat.MPI_TAG == TAG_SPHERE_OBJ_e)   || 
	   (probeStat.MPI_TAG == TAG_NULL_OBJ_e))
	{
	    *pSrcRank = probeStat.MPI_SOURCE;
	    *pRecvTag = probeStat.MPI_TAG;
	    return(YES);
	}else
	{
	    cout << "commMpi2_canRecvObject::get mes but not object" << endl;
	    return(NO);
	}
    }
}

//------------------------------------------------------------
// Is a send process complete?
//   (can send next rays?)
//------------------------------------------------------------
YorN commMpi2_isSendComplete(reqHandle_st req_handle)
{
    reqHandle_st tmp = req_handle;
    int flag = 0;
    static MPI_Status stat;

    MPI_Test(&(tmp.req), &flag, &stat);

    if(flag != 0)
    {
	return(YES);
    }else
    {
	return(NO);
    }
}

//------------------------------------------------------------
// Is a recv process complete?
//------------------------------------------------------------
YorN commMpi2_isRecvComplete(reqHandle_st req_handle)
{
    reqHandle_st tmp = req_handle;
    int flag = 0;
    static MPI_Status stat;

    MPI_Test(&(tmp.req), &flag, &stat);

    if(flag != 0)
    {
	return(YES);
    }else
    {
	return(NO);
    }
}

//------------------------------------------------------------
// send, recv PEStatus to Host
//------------------------------------------------------------
void commMpi2_sendPEstat2Host(PEStatClass* pPEstat)
{
    // Improve commMpi2_sendRay と同様の改良が考えられる
    static MPI_Request requestID;
    static MPI_Status  stat;
    
    MPI_Isend(pPEstat, sizeof(PEStatClass), MPI_BYTE,
	      SysStat::getInstance()->getHostRank(), 
	      TAG_PE_STATUS_e, MPI_COMM_WORLD,
	      &requestID); 
    MPI_Wait(&requestID, &stat);
}

void commMpi2_recvPEstat(PEStatClass *pPEstat)
{
    // Improve commMpi2_sendRay と同様の改良が考えられる
    static MPI_Request requestID;
    static MPI_Status  stat;
    
    MPI_Irecv(pPEstat, sizeof(PEStatClass), MPI_BYTE, 
	      MPI_ANY_SOURCE, TAG_PE_STATUS_e, MPI_COMM_WORLD, &requestID);
    MPI_Wait(&requestID, &stat);
}

