6
$\begingroup$

I need to store large sparse matrices in Eigen. I cannot find anything in the library except the function below, in Eigen/Unsupported. The problem with saveMarket is, that it saves in text format. Due to the size of my matrices I need to store my sparse matrices as binaries. Is there an easy way to adjust the function below to store as a binary. And an easy way to reload the matrix?

template<typename SparseMatrixType>
bool saveMarket(const SparseMatrixType& mat, const std::string& filename, int sym = 0)
{
  typedef typename SparseMatrixType::Scalar Scalar;
  std::ofstream out(filename.c_str(),std::ios::out);
  if(!out)
    return false;

  out.flags(std::ios_base::scientific);
  out.precision(64);
  std::string header; 
  internal::putMarketHeader<Scalar>(header, sym); 
  out << header << std::endl; 
  out << mat.rows() << " " << mat.cols() << " " << mat.nonZeros() << "\n";
  int count = 0;
  for(int j=0; j<mat.outerSize(); ++j)
    for(typename SparseMatrixType::InnerIterator it(mat,j); it; ++it)
    {
    ++ count;
    internal::PutMatrixElt(it.value(), it.row()+1, it.col()+1, out);
    // out << it.row()+1 << " " << it.col()+1 << " " << it.value() << "\n";
}
  out.close();
  return true;
}
Improve this question
$\endgroup$
2
  • $\begingroup$ Why not some standard binary format like hdf5? $\endgroup$
    – Kirill
    Commented Nov 27, 2015 at 18:52
  • $\begingroup$ If you intend to use this code on more than one platform, and especially if you're using a binary file from one machine that was generated on a different machine, you'll have to worry about endianness/byte ordering. As @Kirill suggests, using an already-existing library like HDF5 can save you time, and it will also handle any portability issues. $\endgroup$
    – Daniel Shapero
    Commented Nov 30, 2015 at 18:29

1 Answer 1

Reset to default
6
$\begingroup$

I've rolled my own. Here is a MCVE:

#include <Eigen/Core>
#include <Eigen/Sparse>
#include <iostream>
#include <fstream>
#include <vector>

using namespace Eigen;

typedef Triplet<int> Trip;

template <typename T, int whatever, typename IND>
void Serialize(SparseMatrix<T, whatever, IND>& m) {
    std::vector<Trip> res;
    int sz = m.nonZeros();
    m.makeCompressed();

    fstream writeFile;
    writeFile.open("matrix", ios::binary | ios::out);

    if(writeFile.is_open())
    {
        IND rows, cols, nnzs, outS, innS;
        rows = m.rows()     ;
        cols = m.cols()     ;
        nnzs = m.nonZeros() ;
        outS = m.outerSize();
        innS = m.innerSize();

        writeFile.write((const char *)&(rows), sizeof(IND));
        writeFile.write((const char *)&(cols), sizeof(IND));
        writeFile.write((const char *)&(nnzs), sizeof(IND));
        writeFile.write((const char *)&(innS), sizeof(IND));
        writeFile.write((const char *)&(outS), sizeof(IND));

        writeFile.write((const char *)(m.valuePtr()),       sizeof(T  ) * m.nonZeros());
        writeFile.write((const char *)(m.outerIndexPtr()),  sizeof(IND) * m.outerSize());
        writeFile.write((const char *)(m.innerIndexPtr()),  sizeof(IND) * m.nonZeros());

        writeFile.close();
    }
}

template <typename T, int whatever, typename IND>
void Deserialize(SparseMatrix<T, whatever, IND>& m) {
    fstream readFile;
    readFile.open("matrix", ios::binary | ios::in);
    if(readFile.is_open())
    {
        IND rows, cols, nnz, inSz, outSz;
        readFile.read((char*)&rows , sizeof(IND));
        readFile.read((char*)&cols , sizeof(IND));
        readFile.read((char*)&nnz  , sizeof(IND));
        readFile.read((char*)&inSz , sizeof(IND));
        readFile.read((char*)&outSz, sizeof(IND));

        m.resize(rows, cols);
        m.makeCompressed();
        m.resizeNonZeros(nnz);

        readFile.read((char*)(m.valuePtr())     , sizeof(T  ) * nnz  );
        readFile.read((char*)(m.outerIndexPtr()), sizeof(IND) * outSz);
        readFile.read((char*)(m.innerIndexPtr()), sizeof(IND) * nnz );

        m.finalize();
        readFile.close();

    } // file is open
}


int main(int argc, char *argv[]){
    int rows, cols;
    rows = cols = 6;
    SparseMatrix<double> A(rows,cols), B;

    std::vector<Trip> trp, tmp;

    trp.push_back(Trip(0, 0, rand()));
    trp.push_back(Trip(1, 1, rand()));
    trp.push_back(Trip(2, 2, rand()));
    trp.push_back(Trip(3, 3, rand()));
    trp.push_back(Trip(4, 4, rand()));
    trp.push_back(Trip(5, 5, rand()));
    trp.push_back(Trip(2, 4, rand()));
    trp.push_back(Trip(3, 1, rand()));

    A.setFromTriplets(trp.begin(), trp.end());
    cout << A.nonZeros() << endl;   // Prints 8
    cout << A.size() << endl;       // Prints 36
    cout << A << endl;              // Prints the matrix along with the sparse matrix stuff

    Serialize(A);

    Deserialize(B);

    cout << B.nonZeros() << endl;   // Prints 8
    cout << B.size() << endl;       // Prints 36
    cout << B << endl;              // Prints the reconstructed matrix along with the sparse matrix stuff

    return 0;
}
Improve this answer
$\endgroup$
4
  • 1
    $\begingroup$ That's a nice piece of programming. One small observation: as it exists, I believe that a using namespace std; line is required for this to compile, in general. It might be better to make some small changes to the code to avoid the need for this line (e.g. use std::fstream, etc). $\endgroup$
    – Bill Greene
    Commented Nov 30, 2015 at 19:03
  • $\begingroup$ (-1) It is almost never a good idea to "roll your own" for a task as common and standard as serializing matrices. It can become a serious issue once other people and other software have to interact with a non-standard file format. Furthermore, the example code here has no error-handling at all and even has a hard-coded filename. It is almost always better to use an existing readily-available format, especially since homegrown code would mostly just partially duplicate the functionality of an existing library. $\endgroup$
    – Kirill
    Commented Nov 30, 2015 at 19:45
  • 4
    $\begingroup$ @Kirill I'm not arguing the fact that sometimes it's better to not roll your own, but HDF5 is a pain in the ass to use (IMO). The above code was meant as a guideline for writing the Eigen sparse matrix in binary format, not a product to be shipped to clients or a request on Code Review (hence, no error checking, hard coded filename, etc.). $\endgroup$
    – Avi Ginsburg
    Commented Nov 30, 2015 at 19:55
  • $\begingroup$ @AviGinsburg and others -- please note that Serialize writes out size then in size, but Deserialize reads in size then outside (backwards order). This is a bug and would be great to update the answer to correct it to save future people the debugging time $\endgroup$
    – davewy
    Commented Jan 15, 2020 at 4:40

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.