5
$\begingroup$

I need to store large sparse matrices in Eigen. I cannot find anything in the library except the function below, in Eigen/Unsupported. The problem with saveMarket is, that it saves in text format. Due to the size of my matrices I need to store my sparse matrices as binaries. Is there an easy way to adjust the function below to store as a binary. And an easy way to reload the matrix?

template<typename SparseMatrixType>
bool saveMarket(const SparseMatrixType& mat, const std::string& filename, int sym = 0)
{
  typedef typename SparseMatrixType::Scalar Scalar;
  std::ofstream out(filename.c_str(),std::ios::out);
  if(!out)
    return false;

  out.flags(std::ios_base::scientific);
  out.precision(64);
  std::string header; 
  internal::putMarketHeader<Scalar>(header, sym); 
  out << header << std::endl; 
  out << mat.rows() << " " << mat.cols() << " " << mat.nonZeros() << "\n";
  int count = 0;
  for(int j=0; j<mat.outerSize(); ++j)
    for(typename SparseMatrixType::InnerIterator it(mat,j); it; ++it)
    {
    ++ count;
    internal::PutMatrixElt(it.value(), it.row()+1, it.col()+1, out);
    // out << it.row()+1 << " " << it.col()+1 << " " << it.value() << "\n";
}
  out.close();
  return true;
}
$\endgroup$
2
  • $\begingroup$ Why not some standard binary format like hdf5? $\endgroup$ – Kirill Nov 27 '15 at 18:52
  • $\begingroup$ If you intend to use this code on more than one platform, and especially if you're using a binary file from one machine that was generated on a different machine, you'll have to worry about endianness/byte ordering. As @Kirill suggests, using an already-existing library like HDF5 can save you time, and it will also handle any portability issues. $\endgroup$ – Daniel Shapero Nov 30 '15 at 18:29
4
$\begingroup$

I've rolled my own. Here is a MCVE:

#include <Eigen/Core>
#include <Eigen/Sparse>
#include <iostream>
#include <fstream>
#include <vector>

using namespace Eigen;

typedef Triplet<int> Trip;

template <typename T, int whatever, typename IND>
void Serialize(SparseMatrix<T, whatever, IND>& m) {
    std::vector<Trip> res;
    int sz = m.nonZeros();
    m.makeCompressed();

    fstream writeFile;
    writeFile.open("matrix", ios::binary | ios::out);

    if(writeFile.is_open())
    {
        IND rows, cols, nnzs, outS, innS;
        rows = m.rows()     ;
        cols = m.cols()     ;
        nnzs = m.nonZeros() ;
        outS = m.outerSize();
        innS = m.innerSize();

        writeFile.write((const char *)&(rows), sizeof(IND));
        writeFile.write((const char *)&(cols), sizeof(IND));
        writeFile.write((const char *)&(nnzs), sizeof(IND));
        writeFile.write((const char *)&(innS), sizeof(IND));
        writeFile.write((const char *)&(outS), sizeof(IND));

        writeFile.write((const char *)(m.valuePtr()),       sizeof(T  ) * m.nonZeros());
        writeFile.write((const char *)(m.outerIndexPtr()),  sizeof(IND) * m.outerSize());
        writeFile.write((const char *)(m.innerIndexPtr()),  sizeof(IND) * m.nonZeros());

        writeFile.close();
    }
}

template <typename T, int whatever, typename IND>
void Deserialize(SparseMatrix<T, whatever, IND>& m) {
    fstream readFile;
    readFile.open("matrix", ios::binary | ios::in);
    if(readFile.is_open())
    {
        IND rows, cols, nnz, inSz, outSz;
        readFile.read((char*)&rows , sizeof(IND));
        readFile.read((char*)&cols , sizeof(IND));
        readFile.read((char*)&nnz  , sizeof(IND));
        readFile.read((char*)&inSz , sizeof(IND));
        readFile.read((char*)&outSz, sizeof(IND));

        m.resize(rows, cols);
        m.makeCompressed();
        m.resizeNonZeros(nnz);

        readFile.read((char*)(m.valuePtr())     , sizeof(T  ) * nnz  );
        readFile.read((char*)(m.outerIndexPtr()), sizeof(IND) * outSz);
        readFile.read((char*)(m.innerIndexPtr()), sizeof(IND) * nnz );

        m.finalize();
        readFile.close();

    } // file is open
}


int main(int argc, char *argv[]){
    int rows, cols;
    rows = cols = 6;
    SparseMatrix<double> A(rows,cols), B;

    std::vector<Trip> trp, tmp;

    trp.push_back(Trip(0, 0, rand()));
    trp.push_back(Trip(1, 1, rand()));
    trp.push_back(Trip(2, 2, rand()));
    trp.push_back(Trip(3, 3, rand()));
    trp.push_back(Trip(4, 4, rand()));
    trp.push_back(Trip(5, 5, rand()));
    trp.push_back(Trip(2, 4, rand()));
    trp.push_back(Trip(3, 1, rand()));

    A.setFromTriplets(trp.begin(), trp.end());
    cout << A.nonZeros() << endl;   // Prints 8
    cout << A.size() << endl;       // Prints 36
    cout << A << endl;              // Prints the matrix along with the sparse matrix stuff

    Serialize(A);

    Deserialize(B);

    cout << B.nonZeros() << endl;   // Prints 8
    cout << B.size() << endl;       // Prints 36
    cout << B << endl;              // Prints the reconstructed matrix along with the sparse matrix stuff

    return 0;
}
$\endgroup$
5
  • 1
    $\begingroup$ That's a nice piece of programming. One small observation: as it exists, I believe that a using namespace std; line is required for this to compile, in general. It might be better to make some small changes to the code to avoid the need for this line (e.g. use std::fstream, etc). $\endgroup$ – Bill Greene Nov 30 '15 at 19:03
  • $\begingroup$ (-1) It is almost never a good idea to "roll your own" for a task as common and standard as serializing matrices. It can become a serious issue once other people and other software have to interact with a non-standard file format. Furthermore, the example code here has no error-handling at all and even has a hard-coded filename. It is almost always better to use an existing readily-available format, especially since homegrown code would mostly just partially duplicate the functionality of an existing library. $\endgroup$ – Kirill Nov 30 '15 at 19:45
  • 3
    $\begingroup$ @Kirill I'm not arguing the fact that sometimes it's better to not roll your own, but HDF5 is a pain in the ass to use (IMO). The above code was meant as a guideline for writing the Eigen sparse matrix in binary format, not a product to be shipped to clients or a request on Code Review (hence, no error checking, hard coded filename, etc.). $\endgroup$ – Avi Ginsburg Nov 30 '15 at 19:55
  • $\begingroup$ @AviGinsburg and others -- please note that Serialize writes out size then in size, but Deserialize reads in size then outside (backwards order). This is a bug and would be great to update the answer to correct it to save future people the debugging time $\endgroup$ – davewy Jan 15 '20 at 4:40
  • 1
    $\begingroup$ @davewy Thanks. Fixed. $\endgroup$ – Avi Ginsburg Jan 15 '20 at 9:09

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.