//------------------------------------------------------------
// $Id: host.cc,v 1.9.2.2 2000/06/14 17:46:35 yamauchi Exp $
//------------------------------------------------------------
//	(M\pi)^2 host computer implementation
//	1996/7/5
//	Copyright (C) 1996--1998 YAMAUCHI Hitoshi  山内 斉
//------------------------------------------------------------
#include <stdio.h>
#include <iostream.h>
#include <iomanip.h>
#include <assert.h>
#include "SLList.h"

#include "config.h"
#include "host.h"
#include "Intensity.h"
#include "mpi2ReaderClass.h"
#include "mpi2_funcs.h"
#include "mpi2communication.h"
#include "Option.h"
#include "FileInfo.h"
#include "volumeFuncs.h"
#include "stopwatch.h"
#include "ElapsedTimer.h"
#include "HostComm.h"

#define OUTPUT_VERBOSE_SHARP_PAR_POLYGONS 10000

#ifdef HAVE_NANOSLEEP
// SGI and Linux has the nanosleep function insted of a usleep function.
#include <time.h>
void myUsleep(unsigned int sleeptime)
{
    timespec tm;
    tm.tv_sec  = 0;
    // usleep's tic is 10e-6 but nano is 10e-9
    tm.tv_nsec = sleeptime * 1000; 
    nanosleep(&tm, NULL);
}
#elif HAVE_USLEEP
#include <unistd.h>
void myUsleep(unsigned int sleeptime)
{
    usleep(sleeptime);
}
#else
#  error "There is no usleep. What a pity."
#endif

// for sync mode
#ifdef FOR_STANDALONE

#else
// #include "hostForParallel.h"
#endif

//------------------------------------------------------------
// sleep 
//------------------------------------------------------------
void pollingSleep(void)
{
    //  myUsleep(POLLING_SLEEP_TIME);
}

void retrySleep(void)
{
    myUsleep(RETRY_WAIT_TIME);
}

//------------------------------------------------------------
// get now time
// 
//     struct timeval {
//         int             tv_sec;         /* seconds */
//         int             tv_usec;        /* microseconds */
//     };
// 表示は 32 bit 整数ならば 11 桁あれば十分のはず．
// したがって，printf("%11.6f\n", getNowTime()); ならばとりあえず十分
// だと思う．
// iostream を利用する場合には cout.precision() で現在の精度を取り出し
// て保持し，setprecision にて変更する．たとえば，
//	current_precision = cout.precision();
//	cout << setprecision(new_precision) << getNowTime() << 
//		setprecision(current_precision) << endl;
// のようにする．
//------------------------------------------------------------
double getNowTime(void)
{
    struct timeval tp;
    gettimeofday(&tp, NULL);
    double ret = ((double)(tp.tv_sec)) + ((double)(tp.tv_usec)) / 1000000.0;
    return(ret);
}

//------------------------------
// output watch value of PE 
//    この値は入力待ちなどでは計上されないので並列計算ではあまり意味がない．
//    したがって論文などには使用しない方が良い．
//------------------------------
void outPEwatch(const char *pMes, const StopWatchClass &watch)
{
    if(Option::getInstance()->getVerboseMode() == YES)
    {
	cout << VERBOSEHEAD << " PE:: " << pMes << " " << watch.getUser()
	     << " " << watch.getSystem() << " " << watch.getTotal() << 
	    " : Never use for papers." << endl; 
    }
}

//------------------------------
// output elapsed time of PE using ElapsedTimerClass
//------------------------------
void outPEEtime(const char *pMes, const ElapsedTimerClass &etimer)
{
    int p = cout.precision(); // 出力の精度の現在の値を保持して後で戻す
    cout << NORMALHEAD << " PE:: elapsed time " << 
	setprecision(TIMEOFDAYPREC) << etimer.getTotalTime() << setprecision(p)
	 << " " << pMes << endl;
}

//------------------------------
// output nowtime value of PE 
//------------------------------
void outPERawTime(const char *pMes)
{
    if(Option::getInstance()->getVerboseMode() == YES)
    {
	int p = cout.precision(); // 出力の精度の現在の値を保持して後で戻す
	cout << VERBOSEHEAD << " PE:: raw time " << 
	    setprecision(TIMEOFDAYPREC) << ElapsedTimerClass::getNowRawTime()
	     << setprecision(p) << " " << pMes << endl;
    }
}

//------------------------------------------------------------
// common function output with headder 
//------------------------------------------------------------
const char *genHd(const char *pHead, const IntVec3& PEid)
{
    static char buf[256];
    if(PEid == HostID)
    {
	sprintf(buf, "%s host::", pHead);	
    }
    else if(PEid == IllegalPEID)
    {
	sprintf(buf, "%s illegal PE, Error? :: ", pHead);	
    }
    else
    {
	sprintf(buf, "%s PE:: %d %d %d : ", pHead, PEid.get(IntVec3::X), 
		PEid.get(IntVec3::Y), PEid.get(IntVec3::Z));
    }
    return(buf);
}

const char* genHd(const char *pHead, const int rank)
{
    static char buf[256];    
    sprintf(buf, "%s rank %d : ", pHead, rank);
    return(buf);    
}

// static void outHostNormalMes(const char *pMes)
// {
//     cout << NORMALHEAD << " host:: " << pMes << endl;
// }

//------------------------------
// output verbose message from host
//------------------------------
static void outHostVerboseMes(const char *pMes)
{
    if(Option::getInstance()->getVerboseMode() == YES)
    {
	cout << VERBOSEHEAD << " host:: " << pMes << endl;
    }
}

//------------------------------
// output host command for ray generation message
//------------------------------
static void outHostComForRayGenMessage(const char* pCommand, int requestPErank)
{
    if(Option::getInstance()->getVerboseMode() == YES)
    {
	int p = cout.precision(); // 出力の精度の現在の値を保持して後で戻す
	cout << VERBOSEHEAD   << " host :: broad " << pCommand 
	     << " ray gen. req PE r = " 	   << requestPErank 
	     << " time " << setprecision(TIMEOFDAYPREC)  
	     << getNowTime() << setprecision(p) << endl;
    }
}

//------------------------------------------------------------
// 現在時刻をメッセージ付きで出力する
//------------------------------------------------------------
static void hostOutNowTime(const char *pMes)
{
    int p = cout.precision(); // 出力の精度の現在の値を保持して後で戻す
    cout << NORMALHEAD << " host:: " << SysStat::getInstance()->getHostRank() 
         << " time " << setprecision(TIMEOFDAYPREC) 
	 << getNowTime() << setprecision(p) << " " << pMes << endl;
}

//------------------------------------------------------------
// protected functions
//------------------------------------------------------------
// check space information and material data 
//------------------------------
void hostComputerClass::
checkSpcData(void)
{
    if(this->mSysInfo.getAllocationScheme() == Random_Alloc)
    {
	const int nspace = 
	    this->mSpcParam.getSubDivNum() *
	    this->mSpcParam.getSubDivNum() *
	    this->mSpcParam.getSubDivNum();
	
	if(this->mSysInfo.getTotalPENum() > nspace)
	{
	    cerr << "Error! : number of subspaces < number of PEs. "
		 << "This is meaningless for parallel processing. "
		 << endl;
	    cerr << "PEs = " << this->mSysInfo.getTotalPENum() 
		 << ", subspaces = " << nspace << endl;
	    commMpi2_exit(1);	    
	}
    }else
    {
	IntVec3 penum = this->mSysInfo.getPEDim();
	int subdivnum = this->mSpcParam.getSubDivNum();
	// 分配可能か (大きさ)
	if((subdivnum < penum.get(IntVec3::X)) || 
	   (subdivnum < penum.get(IntVec3::Y)) ||
	   (subdivnum < penum.get(IntVec3::Z)))
	{
	    cerr << "Error! : space subdivision number is smaller than " << 
		"number of PEs." << endl;
	    cerr << "Numver of PEs = " << penum << ", number of space " <<
		"subdivision " << subdivnum << endl;
	    commMpi2_exit(1);
	}
	// 分配可能か (割り切れるか)
	if(((subdivnum % penum.get(IntVec3::X)) != 0) || 
	   ((subdivnum % penum.get(IntVec3::Y)) != 0) ||
	   ((subdivnum % penum.get(IntVec3::Z)) != 0))
	{
	    cerr << "Error! : space subdivision number can not divede " << 
		"by number of PEs." << endl;
	    cerr << "Numver of PEs = " << penum << ", number of space " <<
		"subdivision " << subdivnum << endl;
	    commMpi2_exit(1);
	}
    }
}

//------------------------------
// set spc2PE
//	空間と PE のマッピングを生成する
//------------------------------
void hostComputerClass::
mapSpacePE(void)
{
    // 分配のために空間と PE のマッピングを作成する
    switch(this->mSysInfo.getAllocationScheme())
    {
    case Block_Alloc:
    {
	spc2PEBlockClass *pS2p = new spc2PEBlockClass;
	pS2p->init(this->mSysInfo.getPEDim(), this->mSpcParam.getSubDivNum(), 
		   HostID);
	this->mpSpace2PE = pS2p;
	outHostVerboseMes("readData : allocated spc2PE Block");
    }
        break;
	
    case Distributed_Alloc:
    {
	spc2PEDistributeClass *pS2p = new spc2PEDistributeClass;
	pS2p->init(this->mSysInfo.getPEDim(), this->mSpcParam.getSubDivNum(),
		   HostID);
	this->mpSpace2PE = pS2p;
	outHostVerboseMes("readData : allocated spc2PE Distributed");
    }
	break;
	
    case Skewed_Alloc:
    {
	spc2PESkewClass *pS2p = new spc2PESkewClass;
	pS2p->init(this->mSysInfo.getPEDim(), this->mSpcParam.getSubDivNum(),
		   HostID);
	this->mpSpace2PE = pS2p;
	outHostVerboseMes("readData allocated spc2PE Skewed-Distributed");
    }
	break;
	
    case Random_Alloc:
    {
	spc2PERandomClass *pS2p = new spc2PERandomClass;
	pS2p->init(this->mSysInfo.getPEDim(), this->mSpcParam.getSubDivNum(),
		   HostID);
	this->mpSpace2PE = pS2p;
	outHostVerboseMes("readData : allocated spc2PE Pseudo-Random");
    }
	break;
	
    default:
	this->mpSpace2PE = NULL;
	cerr << "Error! : no such allocation scheme " << 
	    "hostComputerClass::readData" << endl;
	commMpi2_exit(1);
	break;
    }
}

//------------------------------
// read space information 
//------------------------------
void hostComputerClass::
readSpcData(const char  *pSpcFname)
{
    if(readSpcparamFile(pSpcFname, &(this->mSpcParam)) == NO)
    {
	cerr << "Error! : fail reading spcfile " << pSpcFname << endl;
	commMpi2_exit(1);
    }
    if(Option::getInstance()->getVerboseMode() == YES)
    {
	const char *pHead = genHd(VERBOSEHEAD, HostID);
	cout << pHead << "----- read space param succeeded." << endl;
	this->mSpcParam.print(pHead);
    }
}

//------------------------------
// read sysinfo and deliver to system
//------------------------------
void hostComputerClass::
readSysinfo(const char  *pSysinfoFname)
{
    if(readSysinfoFile(pSysinfoFname, &(this->mSysInfo)) == NO)
    {
	cerr << "Error! : fail to read the sysinfo file." << endl;
	commMpi2_exit(1);
    }else
    {
	if(Option::getInstance()->getVerboseMode() == YES)
	{
	    const char *pHead = genHd(VERBOSEHEAD, HostID);
	    cout << pHead << "----- read sysinfo succeeded." << endl;
	    this->mSysInfo.print(pHead);
	}
    }
    IntVec3 pe = (this->mSysInfo.getPEDim());
    int penum  = pe.get(IntVec3::X) * pe.get(IntVec3::Y) * pe.get(IntVec3::Z);
    cout << NORMALHEAD << " host:: systemdim " << getSystemDimension(pe) << 
	" totalPEnum " << penum << " PEtopology " << pe << endl;

    if(this->getRank() != penum)
    {
	cout << NORMALHEAD << " host:: Error! rank(" << this->getRank() << 
	    ") and PEnum(" << penum << ") is not match. abort." << endl;
	commMpi2_exit(1);	
    }
}

//------------------------------
// read viewdata  
//------------------------------
void hostComputerClass::
readViewdata(const char *pViewdataFname)
{
    if(readViewdataFile(pViewdataFname, &(this->mViewData)) == NO)
    {
	cerr << "Error! : fail read viewdata." << endl;
	commMpi2_exit(1);
    }else
    {
	if(Option::getInstance()->getVerboseMode() == YES)
	{
	    const char *pHead = genHd(VERBOSEHEAD, HostID);
	    cout << pHead << "----- read viewdata succeeded." << endl;
	    this->mViewData.print(pHead);
	}	    
    }
}

//------------------------------------------------------------
// read and deliver material data
//------------------------------------------------------------
// read material data
//   material の情報を読み、名前で sort して host のメンバとして保持す
//   る 
//------------------------------
void hostComputerClass::
readMatData(const char *pMatFname)
{
    assert(pMatFname != NULL);
    SLList < materialClass * > matPtrList;
    if(readMatInfo(pMatFname, &matPtrList) == NO)
    {
	cerr << "Error! : fail reading matfile "<< pMatFname << endl;
	commMpi2_exit(1);	
    }
    
    // sort material information
    const int matArySize = matPtrList.length();
    // ここで new したものをずっと利用する
    materialClass *pMatArray = new materialClass [matArySize];
    int i, j;
    for(i = 0; i < matArySize; i++)
    {
	materialClass *pTmpMat = matPtrList.front();
	pMatArray[i] = (*pTmpMat);
	matPtrList.del_front();
	delete pTmpMat;		// matPtrList の内容のものは new されている
    }
    
    int minMatIdx = 0;
    materialClass swapArea;
    // 単純選択法で sort する
    for(i = 0; i < (matArySize - 1); i++)
    {
	// i は処理の時に入れ換える所 (処理の進み具合)
	minMatIdx = i;
	for(j = (i + 1); j < matArySize; j++)	
	{
	    // i+1 から後で最小のものを選ぶ
	    if(strncmp(pMatArray[j].getName(), 
		       pMatArray[minMatIdx].getName(), 
		       MAX_MATERIAL_NAME_LEN) < 0)
	    {
		minMatIdx = j;
	    }
	}
	// i + 1 から後で最小のものは minMatIdx が指している
	swapArea     = pMatArray[i];
	pMatArray[i] = pMatArray[minMatIdx];
	pMatArray[minMatIdx] = swapArea;
    }

    // material 情報をメンバに代入する
    this->mMatArray.init();
    this->mMatArray.setLength(matArySize);
    for(i = 0; i < matArySize; i++)
    {
	this->mMatArray.set(&(pMatArray[i]), i);
    }
    
    if(Option::getInstance()->getShowDataMode() == YES)
    {
	cout << VERBOSEHEAD << "[========== Readed Data ==========]" << endl;
	cout << VERBOSEHEAD << "materials" << endl;
	for(i = 0; i < matArySize; i++)
	{
	    cout << "[" << i << "] " << pMatArray[i] << endl;
	}
    }
}

//------------------------------
// deliver material data : IMPROVE まとめ送りが可能
//------------------------------
void hostComputerClass:: 
deliverMaterial(void)
{
    assert(FileInfo::getInstance()->getMatFileName() != INIT_FILENAME);
    int i;
    int maxArray = this->mMatArray.getLength();
    for(i = 0; i < maxArray; i++)
    {
	materialClass *pMat = this->mMatArray.get(i);
	commMpi2_BroadMaterial(pMat);	
    }
}

//------------------------------
// broadcast null material data (material data 送信終了)
//------------------------------
void hostComputerClass::
broadNullMaterial(void)
{
    materialClass nullMat;
    commMpi2_BroadMaterial(&nullMat); // material 送信終了を全員に送信する
}

//------------------------------
// read and deliver material 
//------------------------------
void hostComputerClass::
readAndDeliverMaterialData(void)
{
    // read material and geometry, and deliver to PEs
    if(FileInfo::getInstance()->getMatFileName() != INIT_FILENAME)
    {
	this->readMatData((const char*)(FileInfo::getInstance()->
					getMatFileName()));
	this->deliverMaterial();
	outHostVerboseMes("broadcasting material... done.");
    }
    this->broadNullMaterial();
}

//------------------------------
// read geometry data and deliver them to PEs 
//	読み込みながら送信する
//	現在は一つづつ送信する : IMPROVE まとめ送信
//	物体データを1つでも読み込めたら YES を返す
//------------------------------
YorN hostComputerClass::
readAndDeliverGeoData(const char *pGeoFname)
{
    assert(this->mObjInSpcInfoArray.isInited()); //副空間物体数管理表は存在する
    assert(pGeoFname != NULL);

    // material name の表と material の表を作成する
    const int matAryLen 	= this->mMatArray.getLength();
    char    **ppMatName		= new char *[matAryLen];
    materialClass *pMatArray 	= new materialClass[matAryLen];
    int i;
    for(i = 0; i < matAryLen; i++)
    {
	ppMatName[i] = strnew((this->mMatArray.get(i))->getName());
	pMatArray[i] = *(this->mMatArray.get(i));
    }

    // open geometry file
    geoReaderClass geoRead;
    if(geoRead.open(pGeoFname, matAryLen, ppMatName, pMatArray) == NO)
    {
	cerr << "Error! : can't open geometry file [" <<  pGeoFname << endl;
	commMpi2_exit(1);
    }
    
    // read geometry file and deliver geometry info to PEs
    objBaseClass *pObj = NULL;		
    int 	  tkn;
    int   	  spcDevNum = this->mSpcParam.getSubDivNum();
    Vector3 spcPartMin = this->mSpcParam.getPartVolumeMin();
    Vector3 spcPartMax = this->mSpcParam.getPartVolumeMax();
    Vector3 	  subSpcSize = (spcPartMax - spcPartMin) / double(spcDevNum);
    int numOfReadObject = 0;	// 2^31 以上の物体は読み込めないだろう
				// から numOfReadObject は int で良いと思う 

    while((tkn = geoRead.getOneGeoData(&pObj)) != EOF_tkn)
    {
	// check read data
	if(Option::getInstance()->getShowDataMode() == YES)
	{
	    pObj->print(VERBOSEHEAD);
	}
	if(pObj->checkSpcPos(spcPartMin, subSpcSize, spcDevNum) == NO)
	{
	    cout << "Error! : inconsistency occur between spcpos and object"  
		" at line " << geoRead.getReadLineNum() << endl;
	}

	// どの空間にいくつの object があるかを数える。
	objectNumInASpaceClass onias = 
	    (this->mObjInSpcInfoArray.get(pObj->getSpcPos()));
	onias.mGeoNum++;
	this->mObjInSpcInfoArray.set(pObj->getSpcPos(), onias);

	// deliver object
	static IntVec3 PEid;
	PEid = this->mpSpace2PE->getPEIDfromSpcPos(pObj->getSpcPos());
	commMpi2_sendObject(this->getRank(PEid), pObj);
	numOfReadObject++;

	if((Option::getInstance()->getVerboseMode() == YES) &&
	   (numOfReadObject % OUTPUT_VERBOSE_SHARP_PAR_POLYGONS == 0))
	{
	    cerr << "#";
	}

	// new された object (triangle, sphere) の delete 
	// IMPROVE object のつかいまわしが考えられるが、その場合には、
	// geoRead.getOneGeoData の見直しが必要
	delete pObj;		
#ifndef NDEBUG
	pObj = NULL;		// for safety
#endif
    }
    geoRead.close();		
    if(Option::getInstance()->getVerboseMode() == YES)
    {
	cout << genHd(VERBOSEHEAD, HostID) << 
	    "total number of geometry data = " << numOfReadObject << endl;
    }
    assert(numOfReadObject >= 0);
    
    if(numOfReadObject > 0)
    {
	return(YES);		
    }else
    {
	return(NO);	    
    }
}

//------------------------------
// 物体存在情報の配布
//   素朴な skip バージョン
//------------------------------
void hostComputerClass::
deliverObjInSpaceInfoSoboku(void)
{
    assert((this->mSysInfo.getRayPropSkipMethod() == NoSkip_rayprop) || 
	   (this->mSysInfo.getRayPropSkipMethod() == SobokuSkip_rayprop));
    array3dClass< objectExistInASpaceClass > objExistMap;
    int size1d[3];
    size1d[0] = size1d[1] = size1d[2] = this->mSpcParam.getSubDivNum();
    objExistMap.initBySizeWithBaseZero(size1d);
    
    assert(this->mObjInSpcInfoArray.getTotalSize() ==
	   objExistMap.getTotalSize());

    int idx;
    for(idx = 0; idx < this->mObjInSpcInfoArray.getTotalSize(); idx++)
    {
	// 物体の数の情報を存在の情報だけに変換する
	objectNumInASpaceClass onias = 
	    this->mObjInSpcInfoArray.getAsOneDimArray(idx);
	objectExistInASpaceClass oeias;
	oeias.set(onias.mGeoNum, onias.mVolNum);
	
	// 存在情報をセット
	objExistMap.setAsOneDimArray(idx, oeias);
    }	

    // 送信するためにクラスでなく，基本データ型の配列にする
    OBJINSPC_IMPLEMENT_TYPE *pObjInfoArray = 
	new OBJINSPC_IMPLEMENT_TYPE[objExistMap.getTotalSize()];
    for(idx = 0; idx < objExistMap.getTotalSize(); idx++)
    {
	pObjInfoArray[idx] = (objExistMap.getAsOneDimArray(idx)).getObjInfo();
    }

    commMpi2_BroadObjInSpc(sizeof(OBJINSPC_IMPLEMENT_TYPE) * 
			   objExistMap.getTotalSize(),
			   pObjInfoArray); 
    delete [] pObjInfoArray;
}


//------------------------------
// object data を読み込む
//	副空間中の物体数を数える
// 		geometry data の読み込みと配送
//		volume data の読み込みと配送
//------------------------------
void hostComputerClass::
readAndDeliverObjectData(void)
{
    // 各空間にいくつ物体が存在するかを数える配列を用意
    int size1d[3];
    size1d[0] = size1d[1] = size1d[2] = this->mSpcParam.getSubDivNum();
    this->mObjInSpcInfoArray.initBySizeWithBaseZero(size1d);
    
    //------------------------------------------------------------
    // geometry data の読み込み (file が存在した時のみ)
    //------------------------------------------------------------
    YorN isThereGeoObject = NO;
    if(FileInfo::getInstance()->getGeoFileName() != INIT_FILENAME)
    {
	isThereGeoObject = 
	this->readAndDeliverGeoData((const char*)(FileInfo::getInstance()->
						  getGeoFileName()));
    }
    this->broadNullObject();	// 読み込み終了を示すobjectを全PEへ送信
    
    //------------------------------------------------------------
    // volume data の読み込み (file が存在した時のみ)
    //------------------------------------------------------------
    int numOfTotalReadVols = 0;
    if(FileInfo::getInstance()->getVolumeInFileListName() != INIT_FILENAME)
    {				
	// ref.: volumeReader.cc
	// この形のように object in space array を渡す形が良い
	const char *pVolInFList = 
	    FileInfo::getInstance()->getVolumeInFileListName();
	numOfTotalReadVols = readAllVolumeData(pVolInFList, this->mSpcParam,
					       &(this->mObjInSpcInfoArray));
    }
    this->broadNullVolume(numOfTotalReadVols); // 読み込み終了を各 PE へ示す
    
    // 物体が一つも存在しない場合にはレンダリングする必要がない．
    if((isThereGeoObject == NO) && (numOfTotalReadVols == 0))
    {
	cout << "There is No object. no need to render. exit." << endl;
	commMpi2_exit(1);
    }

    //------------------------------
    // 物体存在表の配布
    //------------------------------
    switch(this->mSysInfo.getRayPropSkipMethod())
    {
    case NoSkip_rayprop: // skip なしでも物体情報の統計のために deliver する
    case SobokuSkip_rayprop:
	this->deliverObjInSpaceInfoSoboku();
	break;
    case ProximityClouds_rayprop:
	this->PRC_deliverSkipInfoMap(FileInfo::getInstance()->
				     getDistanceDataFileName());
	break;
    default:
	cout << "Internal Error!: host::readAndDeliverObjectData" << endl;
	commMpi2_exit(1);
	break;
    }

    // 表の内容を出力する
    if(Option::getInstance()->getShowNumobjInSpace() == YES) 
    {
	this->printObjInSpcMap();
    }
}

//------------------------------
// broadcast NULLobject to terminate send-receive objects
//------------------------------
void hostComputerClass::
broadNullObject(void)
{
    int x, y, z;
    objBaseClass *pNullObject = new nullObjectClass;
    for(z = 0; z < (this->mSysInfo.getPEDim()).get(IntVec3::Z); z++)
    {
	for(y = 0; y < (this->mSysInfo.getPEDim()).get(IntVec3::Y); y++)
	{
	    for(x = 0; x < (this->mSysInfo.getPEDim()).get(IntVec3::X); x++)
	    {
		commMpi2_sendObject(this->getRank(IntVec3(x, y, z)), 
				    pNullObject); 
	    }
	}
    }
    delete pNullObject;	
}

//------------------------------
// save result
//------------------------------
void hostComputerClass:: 
saveResult(const char *pOutFile)
{
    if(Option::getInstance()->getVerboseMode() == YES)
    {
	cout << VERBOSEHEAD << " host :: save file is " << pOutFile << endl;
    }
    this->mGlobalFrameBuffer.saveData(pOutFile);
}

//------------------------------
// 全ての PE が稼働中のようにセットする
//------------------------------
void hostComputerClass::
initPEStatAllMoving(void)
{
    assert(SysStat::getInstance()->getCommSize() > 1);
    if(this->mPEStats == NULL)
    {
	this->mPEStats = 
	    new PEStatClass[SysStat::getInstance()->getCommSize() - 1];
    }
    int i;
    // PEはrank [0, size - 1]
    for(i = 0; i < (SysStat::getInstance()->getCommSize() - 1); i++)
    {	
	this->mPEStats[i].setBusy(i);
    }
}

//------------------------------
// PE 状態の記録の更新
//------------------------------
void hostComputerClass::
updatePEstat(const PEStatClass& PEstat)
{
    int PErank = PEstat.getSrcRank();
    assert((0 <= PErank) && 
	   (PErank < SysStat::getInstance()->getCommSize() - 1));
    this->mPEStats[PErank] = PEstat;

#ifndef REDUCE_VERBOSE_INFO
    if(Option::getInstance()->getDebugMode() == YES)
    {
	int i; 
	cout << DEBUGHEAD << " host :: ";
	for(i = 0; i < SysStat::getInstance()->getCommSize() - 1; i++)
	{
	    cout << i << " ";
	    this->mPEStats[i].print(); 
	    cout << " ";
	}
	cout << endl;
    }
#endif
}

//------------------------------
// analyze PE information and action 
//------------------------------
void hostComputerClass::
analyzePEInfo(const PEStatClass& PEstat)
{
    if(PEstat.getStat() != PESt_ELSE_e)
    {
	// PE の status 報告
	//    system 動作状況を更新する
	this->updatePEstat(PEstat);
    }else
    {
	// PE からの要求
	if(PEstat.getRequest() == PEReq_Request_StopGenRays_e)
	{
	    if(this->mReqNumOfPEStopRays == 0) // 初の停止要求
	    {
		// システムにレイ発生停止要求
		HostCommClass hcom = 
			HostCommClass(HOSTCOM_Suspend_GenNewRays_e);
		this->broadHostCommand(hcom);
		outHostComForRayGenMessage("suspend", PEstat.getSrcRank());
	    }
	    this->mReqNumOfPEStopRays++;
	    // cout<<"debug: from " << PEstat.getSrcRank() <<
	    // " get stop ray request " << this->mReqNumOfPEStopRays << endl;
	}
	else if(PEstat.getRequest() == PEReq_Request_ResumeGenRays_e)
	{
	    if(this->mReqNumOfPEStopRays == 1) // 最後の再開要求
	    {
		// システムにレイ発生再開要求
		HostCommClass hcom = 
			HostCommClass(HOSTCOM_Resume_GenNewRays_e);
		this->broadHostCommand(hcom);
		outHostComForRayGenMessage("resume",  PEstat.getSrcRank());
	    }
	    this->mReqNumOfPEStopRays--;
	    // cout << "debug: from " << PEstat.getSrcRank() << 
	    // " resum stop ray request " << this->mReqNumOfPEStopRays << endl;
	}
	assert(this->mReqNumOfPEStopRays >= 0);
    }
}

//------------------------------
// 1つでも動作中の PE があるか
//------------------------------
YorN hostComputerClass::
movingPE(void)
{
    int i;
    // PE の rank は [0, size - 1]
    for(i = 0; i < (SysStat::getInstance()->getCommSize() - 1); i++) 
    {	
	if(this->mPEStats[i].getStat() == PESt_BUSY_e)
	{
	    return(YES);
	}
    }
    return(NO);    
}

//------------------------------
// Host command を全 PE へ送信
//	(MPI には non blocking broadcast がない)
// host command まで来てしまったら ray-trace や radiosity は抜けてしまっ
// ているので同期は barrierSync で取る
//------------------------------
void hostComputerClass::
broadHostCommand(const HostCommClass& hostcom)
{
    int i;
    for(i = 0; i < (SysStat::getInstance()->getCommSize() - 1); i++)
    {
	if(Option::getInstance()->getDebugMode() == YES)
	{
	    cout << genHd(DEBUGHEAD, this->getRank())<< " broad host command "
	    	 << this->getRank() << " to " << i << "\t" <<
		 (hostcom.toString()) << endl; 
	}
	HostCommClass *pHostComm = new HostCommClass(hostcom);
	this->mSendHostCommBuf.enqNewedObjwithMaySend(pHostComm, i);
    }
}

 
//------------------------------
// show object in subspaces
//------------------------------
void hostComputerClass::
printObjInSpcMap(void) const
{
    printObjInSpcMapCommon("## host::", this->mObjInSpcInfoArray);
}

//------------------------------
// consistency check
//	data file の一貫性について
//	気がついたことを check することにする
//------------------------------
void hostComputerClass::
consistencyCheck(void)
{
    // warning error
    if(this->mSysInfo.getFirstRaysGenerationMethod() == AllScan_frgm)
    {
	if(this->mViewData.getDistributedRayNum() > 1)
	{
	    const char * mes[] = {
// 		"Warning! : When First Ray Generation is `ALLScan' mode.",
// 		"\t rays are generated but not distributed on a pixel.",
// 		"`ALLScan' mode 時には 1 pixel に複数の ray を発っした場合，",
// 		"並列動作においては正しい画像が得られません．",
// 		"ある pixel のランダムな位置に ray が発射された場合に，その ",
// 		"ray がその担当空間のものかを決定する際，全部を scan して取",
// 		"捨選択するアルゴリズムでは，各 PE で同一の pixel にはまった",
// 		"く同じ ray のサンプリングが必要ですが，その実現には隣接空間",
// 		"を担当する PE 間での交渉等が必要となります．これは ALLScan ",
// 		"特有の問題で他の分散初期光発生では自然に解決できるので，",
// 		"Allscan の場合だけ余計なことをするのは比較に対して公平でな",
// 		"いとの判断からそのままにしてあります．",
		NULL,
	    };
	    for(int i = 0; mes[i] != NULL; i++)
	    {
		cout << NORMALHEAD << " " << mes[i] << endl;
	    }
	}
    }

    // exit error
    if(this->mSysInfo.isRayTracingCalc() == NO)
    {
	if(this->mViewData.getDistributedRayNum() > 1)
	{
	    cout << "Error! : ray tracing mode is off."	<< endl;
	    cout << "\t DistributedRayNum must be 1 instead of " << 
		this->mViewData.getDistributedRayNum() << endl;
	    commMpi2_exit(1);
	}
    }
}

//------------------------------------------------------------
// public functions
//------------------------------------------------------------
hostComputerClass::
hostComputerClass(void) 
{
    // `:' によるメンバ(初期値) の初期化は可読性が悪いのでやめる．
    // host は一度しか construct されないので問題ない.

    this->mRank = -1;
    //     this->mSysInfo; 
    //     this->mViewData;
    //     this->mSpcParam;
    //     this->mMatArray;

    this->mpSpace2PE		= NULL;
    
    // 	   this->mObjInSpcInfoArray;
    //     this->mGlobalFrameBuffer;
    
    this->mPEStats		= NULL;
    this->mReqNumOfPEStopRays	= 0;
    // this->mIntPtrQueue;
    // this->mIntRecvBuff;
    
    // for Proximity Clouds
    this->mpDistanceDataFileName	= NULL;
}

void  hostComputerClass::
init(int hostRank)
{
    assert(hostRank > 0);	// 0 も可能であるが、その場合には PE がいない
    this->mRank	= hostRank; 

    const FileInfo *pFI = FileInfo::getInstance();
    if(pFI->isOKforMpi2() == NO)
    {
	cout << "Error! : host::init : FileInfo has lack of consistency" 
	<< endl;
	commMpi2_exit(1);	
    }

    // output object size
    if(Option::getInstance()->getVerboseMode() == YES)
    {
	outputObjectSize(VERBOSEHEAD);
    }

    //---------- open check 
    const char *pErrMsg = "Error! : can't open ";
    YorN isOK = YES;
    FILE *pfp = NULL;
    if((pfp = fopen((const char*)(pFI->getSpcFileName()), "r")) == NULL)
    {
	isOK = NO;
	cout << pErrMsg << "spc file [" <<pFI->getSpcFileName()<< "]" << endl;
    }else
    {
	fclose(pfp);
    }
    if((pfp = fopen((const char*)(pFI->getSysInfoFileName()), "r")) == NULL)
    {
	isOK = NO;
	cout << pErrMsg << "sysinfo file [" << pFI->getSysInfoFileName()
		<< "]" << endl;
    }else
    {
	fclose(pfp);
    }
    if((pfp = fopen((const char*)(pFI->getViewDataFileName()), "r")) == NULL)
    {
	isOK = NO;
	cout << "Error! : can't open viewdata file [" << 
		pFI->getViewDataFileName() << "]" << endl;
    }else
    {
	fclose(pfp);
    }
    if((pfp = fopen((const char*)(pFI->getSaveResultFileName()), "r")) != NULL)
    {
	isOK = NO;
	cout << "Error! : there are save file [" << 
		pFI->getSaveResultFileName() << "]" << endl;
	fclose(pfp);
    }


    if(pFI->getMatFileName() != INIT_FILENAME)
    {
	if((pfp = fopen((const char*)(pFI->getMatFileName()), "r")) == NULL)
	{
	    isOK = NO;
	    cout << pErrMsg << "mat file ["<<pFI->getMatFileName()<<"]"<< endl;
	}else
	{
	    fclose(pfp);
	}

	if(strncmp((const char*)(pFI->getGeoFileName()), 
		   "(stdin)", strlen("(stdin)")) == 0)
	{
	    // nothing to do : geometry file のみ stdin からの読み込みを許す
	}
	else
	{ 
	    if((pfp = fopen((const char*)(pFI->getGeoFileName()), "r")) 
	       == NULL)
	    {
		isOK = NO;
		cout << pErrMsg << "geo file [" 
		     << pFI->getGeoFileName() << "]" << endl;
	    }else
	    {
		fclose(pfp);
	    }
	}
    }
    // volume infile を指定している場合
    if(pFI->getVolumeInFileListName() != INIT_FILENAME) 
    {
	if((pfp = fopen((const char*)(pFI->getVolumeInFileListName()), "r")) 
	   == NULL)
	{
	    isOK = NO;		// 指定しているにもかかわらず開けなかった
	    cout << "Error! : can't open volume infile [" << 
		pFI->getVolumeInFileListName() << "]" << endl;
	}else
	{
	    fclose(pfp);
	}
    }

    // for PRC method
    if(pFI->getDistanceDataFileName() != INIT_FILENAME)
    {
	if((pfp = fopen((const char*)(pFI->getDistanceDataFileName()), "r"))
	   == NULL)
	{
	    isOK = NO;		// 指定しているにもかかわらず開けなかった
	    cout << pErrMsg << "map file  [" <<	pFI->getDistanceDataFileName() 
	    << "]" << endl;
	}else
	{
	    fclose(pfp);
	}
    }

    if(isOK == NO)
    {
	cout << "Error! : fail to file open at hostComputerClass::init"<< endl;
	commMpi2_exit(1);
    }

    if(Option::getInstance()->getVerboseMode() == YES)
    {
	char buf[64];
	sprintf(buf, "%s host:: ", VERBOSEHEAD);
	pFI->print(buf);
    }

    this->initNetworkInterfaceUnit();
}

//------------------------------
// read data
//	・sysinfo, material は全員に broadcast される
//	・object は読み込みごとに host から送信される。
//	・各 PE では object を受信し、数えて収納する。
//	・どの {material, object} でもない NULL{material, Object} により、
//	  分配終了を示す 
//------------------------------
void hostComputerClass::
readAndDeliverAllData(void)
{
    // read sysinfo, viewdata, spacedata
    FileInfo *pFI = FileInfo::getInstance();
    this->readSysinfo(pFI->getSysInfoFileName());
    this->readViewdata(pFI->getViewDataFileName());
    this->readSpcData(pFI->getSpcFileName());

    // consistent check
    this->consistencyCheck();

    // broadcast sysinfo, viewdata, spacedata
    commMpi2_BroadSysinfo(&(this->mSysInfo),
			  &(this->mViewData), &(this->mSpcParam));
    outHostVerboseMes("deliver sysinfo, viewdata, spcparam done.");

    // sysinfo, spcdata から SubSpaces <-> PE のマップを生成する。
    this->mapSpacePE();

    // check spcdata
    this->checkSpcData();
    
    // read material and geometry, and deliver to PEs
    this->readAndDeliverMaterialData();

    // read object (geo and/or object) and deliver to PEs
    this->readAndDeliverObjectData();
    outHostVerboseMes("readData ok!");

    // Frame Buffer
    int scrX, scrY;
    this->mViewData.getScreenSize(&scrX, &scrY);
    this->mGlobalFrameBuffer.init(scrX, scrY);
    if(FileInfo::getInstance()->isIntensityImmidiateOut() == YES)
    {
	this->mGlobalFrameBuffer.setImmidiateOut();
	cout << "GFBSIZE " << scrX << " " << scrY << endl; // output GFB size 
    }
}

//------------------------------
// calculate radiosity
//	基本は ray-tracing と同じ
//------------------------------
void hostComputerClass::
doRadiosity(void)
{
    if(this->mSysInfo.isRadiosityCalc() == NO)
    {
	// skip radiosity stage
	outHostVerboseMes("skip radiosity stage.");
	return;
    }

    // not skip the radiosity stage
    hostOutNowTime("start radiosity stage.");
    this->initRadiosityStage();

    int endRetryNum = 0;
    // 通信遅延のために再び busy になるかもしれないのでリトライする。
    while(endRetryNum < MAX_RETRY_NUM_CALC)
    {
	while((this->movingPE() == YES) ||               // 動作中 PE 有り
	      (this->mRecvPEStatBuf.isRecvObj() == YES)) // あるいは受信中
	{
	    // PEStat の受信 : 受信可能な限り
	    this->mRecvPEStatBuf.recv();	
	    while(this->mPEStatRecvPtrQueue.length() > 0)
	    {
		this->mRecvPEStatBuf.recv();	
		assert(this->mPEStatRecvPtrQueue.length() > 0);
		PEStatClass *pPEstat = this->mPEStatRecvPtrQueue.deq();
		this->analyzePEInfo(*pPEstat);
		delete pPEstat; // 受信時に new されている

		this->mHStat.incrGetPEStatsNum(); // 統計情報収集
	    }

	    // 受信はなくなった
	    pollingSleep(); 	// wait for suppress probing load
	    endRetryNum = 0; 	// reset number of retry.
	}

	// 受信もなく，動作中 PE もなくなった．
	if(Option::getInstance()->getVerboseMode() == YES)
	{
	    cout << VERBOSEHEAD << " host:: finish radiosity, but wait." 
	    	" retry " << RETRY_WAIT_TIME << " usec: retrynum " <<
		endRetryNum << endl;
	}
	endRetryNum++;
	this->mRecvPEStatBuf.recv(); // 受信を試みる
	retrySleep();		     // wait for suppress probing load	
    }

    // finalize radiosity stage
    hostOutNowTime("end radiosity stage.");
    HostCommClass hcomm = HostCommClass(HOSTCOM_End_Radiosity_e);
    this->broadHostCommand(hcomm);
    this->mHStat.print();	// 統計情報出力
}

//------------------------------
// prepare shading 
//	基本は ray-tracing と同じ
//------------------------------
void hostComputerClass::
doPrepareShading(void)
{
    if(this->mSysInfo.getShadingScheme() == FlatShading)
    {
	// flat shading
	outHostVerboseMes("flat shading. skip the prepare shading pass.");
	return;
    }

    // not flat shading ... start prepare shading.
    hostOutNowTime("start the prepare shading pass.");
    this->initPEStatAllMoving();
    
    // ここでは idle の後にまた busy になることがないので retry は不要
    while((this->movingPE() == YES) || 		     // 動作中 PE 有
	  (this->mRecvPEStatBuf.isRecvObj() == YES)) // あるいは受信中
    {
	// PEStat の受信 : 受信可能な限り
	this->mRecvPEStatBuf.recv();	
	while(this->mPEStatRecvPtrQueue.length() > 0)
	{
	    assert(this->mPEStatRecvPtrQueue.length() > 0);
	    PEStatClass *pPEstat = this->mPEStatRecvPtrQueue.deq();
	    this->updatePEstat(*pPEstat); // system 動作状況の更新
	    delete pPEstat;		  // 受信時に new されている

	    this->mHStat.incrGetPEStatsNum();
	}

	retrySleep();		// wait for suppress probing load
    }
    hostOutNowTime("done prepare shading pass... send hostcommand to all PE.");
    
    HostCommClass hcom = HostCommClass(HOSTCOM_End_PrepareShading_e);
    this->broadHostCommand(hcom);
}

//------------------------------
// calculate ray-tracing
//------------------------------
void hostComputerClass::
doRayTracing(void)
{
    hostOutNowTime("start the ray tracing stage.");
    this->initRayTraceStage();
    this->mHStat.init();

    int endRetryNum = 0;	// 通信遅延のためにリトライする。その回数
    while(endRetryNum < MAX_RETRY_NUM_CALC)
    {
	while((this->movingPE()	  	        == YES) || // 動作中 PE 有
	      (this->mIntRecvBuff.isRecvObj() 	== YES) || // 輝度受信中
	      (this->mRecvPEStatBuf.isRecvObj() == YES))   // PEStat 受信中
	      
	{
	    // 輝度情報受信キューに輝度がある間とり出す
	    this->mIntRecvBuff.recv(); // 輝度受信		
	    while(this->mIntPtrQueue.length() > 0)
	    {
		assert(this->mIntPtrQueue.length() > 0);
		IntensityClass * pInt = this->mIntPtrQueue.deq();
		this->setIntensity(*pInt);
		delete pInt;
	    }
	    
	    // PEStat 受信キューに PEStat がある
	    this->mRecvPEStatBuf.recv();
	    while(this->mPEStatRecvPtrQueue.length() > 0)
	    {
		this->mRecvPEStatBuf.recv();
		// system 動作状況の更新

		assert(this->mPEStatRecvPtrQueue.length() > 0);
		PEStatClass *pPEstat = this->mPEStatRecvPtrQueue.deq();
		this->analyzePEInfo(*pPEstat);
		delete pPEstat;	// 受信時に new されている
		
		this->mHStat.incrGetPEStatsNum(); // 統計情報収集
	    }
	    endRetryNum = 0;	// reset number of retry.
	    pollingSleep();	// wait for suppress probing load
	}
	if(Option::getInstance()->getVerboseMode() == YES)
	{
	    cout << VERBOSEHEAD << " host:: finish ray trace, but wait." 
		" retry " << RETRY_WAIT_TIME << " usec: retrynum " << 
		    endRetryNum << endl;
	}
	retrySleep();
	endRetryNum++;
    }

    // finalize ray-tracing stage
    hostOutNowTime("end the ray tracing stage.");
    HostCommClass hcom = HostCommClass(HOSTCOM_End_Raytrace_e);
    this->broadHostCommand(hcom);
    this->mHStat.print();	// 統計情報出力
}    

//------------------------------
// out put result image
//------------------------------
void hostComputerClass::
outResult(void)
{
    this->saveResult(FileInfo::getInstance()->getSaveResultFileName());
}

//------------------------------
// PE含むバリア同期 time out あり失敗すると 全体で exit
//   ここに来た時には同期が可能でなくてはいけないが、もしそうでなかっ
//   た場合には exit する 
//------------------------------
void hostComputerClass::
barrierSync(void)
{
    static int syncID = 0;	// 何度目の barrierSync か
    int hostrank = SysStat::getInstance()->getHostRank();

//     cout << NORMALHEAD << " host:: " << hostrank << 
//     	" waiting PEs sync : syncID = " << syncID << endl;
    commMpi2_Barrier();

//     int p = cout.precision(); // 出力の精度の現在の値を保持して後で戻す
//     cout << NORMALHEAD << " host:: " << hostrank << " time "
//     	 << setprecision(TIMEOFDAYPREC) << getNowTime() << 
// 	 setprecision(p) << " sync done. " << syncID << endl;
    syncID++;
}

//------------------------------
// output result image
//------------------------------

//------------------------------------------------------------
// for breakpoint
//------------------------------------------------------------
#ifndef NDEBUG
void forBreakPoint(void)
{
    cout << "This is break point" << endl;
}
#endif
