After some parallel computations, I need to write the results in a file and for plotting reasons, I want them in ASCII format. Because of performance issues, I want to avoid to MPI_SEND the data to the master process and make him print. Thus, I am wondering if there is a way to write directly in ASCII with MPI.
Fortran examples would be excellent.
It's possible to use VTK library and its parallel IO built-in mechanism to write the files from each rank to different file and ParaView could combine them again to show you the visualization. Also, I should say I don't have a FORTRAN example and unfortunately VTK doesn't support FORTRAN officially. So, I'll show you a C++ example, which I'm not sure how much would be useful for you. In order to accomplish that you need to build VTK with MPI enabled flag to have access to vtkMPIController class. I put the whole function as this:
First you need to define your header file named VtkParallelWriter.h:
// MPI Library
#include <mpi.h>
//VTK Library
#include <vtkXMLPStructuredGridWriter.h>
#include <vtkStructuredGrid.h>
#include <vtkSmartPointer.h>
#include <vtkDoubleArray.h>
#include <vtkPointData.h>
#include <vtkMPIController.h>
#include <vtkProgrammableFilter.h>
#include <vtkInformation.h>
struct Args {
vtkProgrammableFilter* pf;
int local_extent[6];
};
void execute (void* arg);
void vtkParallelWriter(int argc, char *argv[],std::vector<double*>colors, std::vector<char*> names, int LX, int LY, int LZ, double x_min, double x_max, double y_min, double y_max, double z_min, double z_max, double local_origin_x, double local_origin_y, double local_origin_z, int nn, int timesnapshot);
Then the main body of this function goes to VtkParallelWriter.cc file:
#include <vector>
#include <string>
#include <iostream>
#include "VtkParallelWriter.h"
// function to operate on the point attribute data
void execute (void* arg) {
Args* args = reinterpret_cast<Args*>(arg);
auto info = args->pf->GetOutputInformation(0);
auto output_tmp = args->pf->GetOutput();
auto input_tmp = args->pf->GetInput();
vtkStructuredGrid* output = dynamic_cast<vtkStructuredGrid*>(output_tmp);
vtkStructuredGrid* input = dynamic_cast<vtkStructuredGrid*>(input_tmp);
output->ShallowCopy(input);
output->SetExtent(args->local_extent);
}
void vtkParallelWriter(int argc, char *argv[],std::vector<double*>colors, std::vector<char*> names, int LX, int LY, int LZ, double x_min, double x_max, double y_min, double y_max, double z_min, double z_max, double local_origin_x, double local_origin_y, double local_origin_z, int nn, int timesnapshot) {
int global_extent[6] = {x_min, x_max, y_min, y_max, z_min, z_max};
bool flagX, flagY, flagZ;
int new_local_origin_x;
int new_local_origin_y;
int new_local_origin_z;
flagX = false;
flagY = false;
flagZ = false;
if (local_origin_x == 0) {
flagX = true;
}
if (local_origin_y == 0) {
flagY = true;
}
if (local_origin_z == 0) {
flagZ = true;
}
if (flagX == false) {
new_local_origin_x = local_origin_x - 1;
}
if (flagY == false) {
new_local_origin_y = local_origin_y - 1;
}
if (flagZ == false) {
new_local_origin_z = local_origin_z - 1;
}
if (flagX == true) {
new_local_origin_x = local_origin_x;
}
if (flagY == true) {
new_local_origin_y = local_origin_y;
}
if (flagZ == true) {
new_local_origin_z = local_origin_z;
}
int local_extent[6] = {new_local_origin_x, local_origin_x+LX-1, new_local_origin_y, local_origin_y+LY-1, new_local_origin_z, local_origin_z+LZ-1};
int dims[3] = {local_origin_x+LX-new_local_origin_x, local_origin_y+LY-new_local_origin_y, local_origin_z+LZ-new_local_origin_z};
// Create and Initialize vtkMPIController
auto contr = vtkSmartPointer<vtkMPIController>::New();
if (timesnapshot == 0) {
contr->Initialize(&argc, &argv, 1);
}
int nranks = contr->GetNumberOfProcesses();
int rank = contr->GetLocalProcessId();
// Create grid points, allocate memory and Insert them
auto points = vtkSmartPointer<vtkPoints>::New();
for (int k=0; k<dims[2]; ++k) {
for (int j=0; j<dims[1]; ++j) {
for (int i=0; i<dims[0]; ++i) {
points->InsertNextPoint(new_local_origin_x+i, new_local_origin_y+j, new_local_origin_z+k);
}
}
}
// Create a density field. Note that the number of cells is always less than
// number of grid points by an amount of one so we use dims[i]-1
std::vector<vtkSmartPointer<vtkDoubleArray>> vtkColors;
for (int iterator = 0; iterator < colors.size(); iterator++) {
auto tempColor = vtkSmartPointer<vtkDoubleArray>::New();
tempColor->SetNumberOfComponents(1);
tempColor->SetName(names[iterator]);
int LXP = nn+LX+nn;
int LYP = nn+LY+nn;
int LZP = nn+LZ+nn;
for (int k=0; k<dims[2]; ++k) {
int K;
if (flagZ == true) {
K = k + nn;
}
if (flagZ == false) {
K = k;
}
for (int j=0; j<dims[1]; ++j) {
int J;
if (flagY == true) {
J = j + nn;
}
if (flagY == false) {
J = j;
}
for (int i=0; i<dims[0]; ++i) {
int I;
if (flagX == true) {
I = i + nn;
}
if (flagX == false) {
I = i;
}
int IDflattened = I+J*LXP+K*LXP*LYP;
tempColor->InsertNextTuple1(colors[iterator][IDflattened]);
}
}
}
vtkColors.push_back(tempColor);
}
// Create a vtkProgrammableFilter
auto pf = vtkSmartPointer<vtkProgrammableFilter>::New();
// Initialize an instance of Args
Args args;
args.pf = pf;
for(int i=0; i<6; ++i) args.local_extent[i] = local_extent[i];
pf->SetExecuteMethod(execute, &args);
// Create a structured grid and assign point data and cell data to it
auto structuredGrid = vtkSmartPointer<vtkStructuredGrid>::New();
structuredGrid->SetExtent(global_extent);
pf->SetInputData(structuredGrid);
structuredGrid->SetPoints(points);
for (int iterator = 0; iterator < vtkColors.size(); iterator++) {
structuredGrid->GetPointData()->AddArray(vtkColors[iterator]);
}
std::string fileName = std::string("./out/output_") + std::to_string(timesnapshot) + ".pvts";
// Create the parallel writer and call some functions
auto parallel_writer = vtkSmartPointer<vtkXMLPStructuredGridWriter>::New();
parallel_writer->SetInputConnection(pf->GetOutputPort());
parallel_writer->SetController(contr);
parallel_writer->SetFileName(fileName.c_str());
parallel_writer->SetNumberOfPieces(nranks);
parallel_writer->SetStartPiece(rank);
parallel_writer->SetEndPiece(rank);
parallel_writer->SetDataModeToBinary();
parallel_writer->Update();
parallel_writer->Write();
}
This example is for 3D structured grids but if you set LZ to 1 and z_min and z_max to 0, it should work for 2D structured grids as well. You basically flatten your variable color in 2D to store in a 1D array defined by double* and if you have some of them you store each of them in a vector of arrays to basically give it to std::vector<double*>colors. This code is tested and verified for 3D structured grids with VTK 8.9 built with MPI enabled.
The problem with output from parallel processes is that they all go through ssh tunnels, and there is no guarantee in what order they will arrive. Even if you use Send/Recv to sequentialize them.
You could do the following:
mpirun -np 8 program_script
where program_script:
#!/bin/bash
your_program > program.out$PMPI_RANK
and then
for i in `seq 1 8` ; do cat program.out$i ; done
This uses the fact that most MPI implementations set an environment variable that is unique for each spawned process.
A drop-in replacement for your method is to have every MPI process determine the number of bytes it is going to write, distribute this information with MPI_Allgather, and write one big file in parallel using this information by MPI_File_open, MPI_File_seek, MPI_File_write, and MPI_File_close.
Think ahead about how to read this file though. It makes sense to design a file format that is partition-independent, in the sense that it can be read by any number of proceses unrelated to the original simulation size. If possible, place necessary header information in the beginning and avoid having each process placing its own header right in front of its data. You can then read the header on one rank and MPI_Bcast it before using independent reads by many processes.
For the special case of VTK, it may not always be practical to write one file per MPI process. I once got a phone call from the supercomputing center because I was accidentally writing 32768 VTU files per second.
