/**********************************************************

This software is part of J.-S. Caux's ABACUS library.

Copyright (c) J.-S. Caux.

-----------------------------------------------------------

File:  LiebLin_DSF_GeneralState_par_Run.cc

Purpose:  Parallel version of ABACUS using MPICH.

***********************************************************/

#include "ABACUS.h"
#include "mpi.h"

using namespace std;
using namespace ABACUS;

int main(int argc, char *argv[])
{
  char whichDSF;
  DP c_int, L, kBT;
  int N, iKmin, iKmax, Max_Secs, paralevel;
  DP target_sumrule = 1.0e+6;  // effectively deactivated here
  bool refine = true;  // always true for parallel mode
  char* Ix2filenameprefix;

  if (argc < 10) { // provide some info

    cout << endl << "Welcome to ABACUS\t(copyright J.-S. Caux)." << endl;
    cout << endl << "Usage of LiebLin_DSF_par executable: " << endl;
    cout << endl << "This function runs ABACUS in parallel mode, starting from a preexisting "
      "serial run (obtained using the LiebLin_DSF executable) using the same model parameters." << endl;
    cout << endl << "Provide the following arguments:" << endl << endl;
    cout << "char whichDSF \t\t Which structure factor should be calculated ?  Options are:  "
      "d for rho rho, g for psi psi{dagger}, o for psi{dagger} psi" << endl;
    cout << "DP c_int \t\t Value of the interaction parameter:  use positive real values only" << endl;
    cout << "DP L \t\t\t Length of the system:  use positive real values only" << endl;
    cout << "int N \t\t\t Number of particles:  use positive integer values only" << endl;
    cout << "char* defaultScanStatename:\t\t file [].Ix2 contains the quantum numbers defining the "
      "AveragingState; used as defaultScanStatename" << endl;
    cout << "int iKmin" << endl << "int iKmax \t\t Min and max momentum integers to scan over:  "
      "recommended values:  -2*N and 2*N" << endl;
    cout << "int paralevel" << endl;
    cout << "rank[i], nr_processors[i] \t rank and nr_processors of each earlier paralevels." << endl;
    cout << "int Max_Secs \t\t Allowed computational time:  (in seconds)" << endl;

    return(0);
  }

  int n = 1;
  whichDSF = *argv[n++];
  c_int = atof(argv[n++]);
  L = atof(argv[n++]);
  N = atoi(argv[n++]);
  Ix2filenameprefix = argv[n++];
  iKmin = atoi(argv[n++]);
  iKmax = atoi(argv[n++]);
  paralevel = atoi(argv[n++]); // paralevel == 1 means that we have one layer of parallelization, so no previous rank and nr_processors to specify
  if (argc != 10 + 2*(paralevel - 1)) ABACUSerror("Wrong nr of arguments in LiebLin_DSF_par_Prepare.");
  Vect<int> rank_lower_paralevels(paralevel - 1);
  Vect<int> nr_processors_lower_paralevels(paralevel - 1);
  for (int i = 0; i < paralevel - 1; ++i) {
    rank_lower_paralevels[i] = atoi(argv[n++]);
    nr_processors_lower_paralevels[i] = atoi(argv[n++]);
  }
  Max_Secs = atoi(argv[n++]);

  if (Max_Secs <= 120) ABACUSerror("Please allow more time in LiebLin_DSF_par_Run.");

  int Max_Secs_used = Max_Secs - 120;

  MPI::Init(argc, argv);

  DP tstart = MPI::Wtime();

  int rank_here = MPI::COMM_WORLD.Get_rank();
  int nr_processors_here = MPI::COMM_WORLD.Get_size();

  // Read the Ix2 from the file:
  Vect<int> Ix2_input(N);
  ifstream Ix2_input_file;
  stringstream filenamestrstream;
  filenamestrstream << Ix2filenameprefix;
  string defaultScanStatename = filenamestrstream.str();
  filenamestrstream << ".Ix2";
  string filenamestr = filenamestrstream.str();
  const char* filename_Cstr = filenamestr.c_str();
  Ix2_input_file.open(filename_Cstr);
  if (Ix2_input_file.fail()) {
    cout << filename_Cstr << endl;
    ABACUSerror("Could not open Ix2 input file in LiebLin_DSF_GeneralState");
  }
  for (int i = 0; i < N; ++i) {
    Ix2_input_file >> Ix2_input[i];
  }

  // Now define the AveragingState
  LiebLin_Bethe_State AveragingState(c_int, L, N);
  AveragingState.Ix2 = Ix2_input;
  AveragingState.Compute_All(true);


  Vect<int> rank (paralevel);
  Vect<int> nr_processors (paralevel);
  for (int i = 0; i < paralevel - 1; ++i) {
    rank[i] = rank_lower_paralevels[i];
    nr_processors[i] = nr_processors_lower_paralevels[i];
  }
  rank[paralevel-1] = rank_here;
  nr_processors[paralevel-1] = nr_processors_here;

  if (nr_processors_here < 2) ABACUSerror("Give at least 2 processors to ABACUS parallel !");

  refine = true;

  // ASSUMPTION:  preexisting files (raw, thr, ...) exist for the run.


  DP tnow = MPI::Wtime();


  if (Max_Secs_used > 0) {

    // Barrier synchronization
    MPI_Barrier (MPI::COMM_WORLD);

    // then everybody gets going on their own chunk !
    Scan_LiebLin (whichDSF, AveragingState, defaultScanStatename, iKmin, iKmax, Max_Secs,
		  target_sumrule, refine, paralevel, rank, nr_processors);

    // Another barrier synchronization
    MPI_Barrier (MPI::COMM_WORLD);

    tnow = MPI::Wtime();

  } // while (tnow - tstart...

  MPI::Finalize();

  return(0);
}