1#ifndef LOSSYSTORAGEDUNE_HH
2#define LOSSYSTORAGEDUNE_HH
5#include <unordered_map>
8#include "dune/grid/common/grid.hh"
9#include "dune/grid/common/entitypointer.hh"
10#include "dune/grid/common/entity.hh"
11#include "dune/grid/io/file/dgfparser/dgfparser.hh"
12#include "dune/istl/matrix.hh"
13#include "dune/istl/bcrsmatrix.hh"
14#include "dune/common/fmatrix.hh"
15#include "dune/common/iteratorfacades.hh"
16#include "dune/istl/matrixindexset.hh"
26template <
class Gr
id,
class CoeffVector>
30 static const int dim = Grid::dimension ;
32 static const int order = 1 ;
38 typedef typename Grid::LevelGridView::IndexSet::IndexType
IndexType;
41 typedef typename Grid::Traits::GlobalIdSet::IdType
IdType;
43 typedef unsigned long int ULong;
45 LossyStorage(
int coarseLevel_,
double aTol_) : ps(nullptr), coarseLevel(coarseLevel_), aTol(aTol_),
46 accumulatedEntropy(0), accumulatedOverhead(0),
47 accumulatedUncompressed(0), accumulatedCompressed(0)
66 return accumulatedEntropy;
72 accumulatedEntropy = 0;
80 return accumulatedOverhead;
85 accumulatedOverhead = 0;
93 return accumulatedCompressed;
98 accumulatedCompressed = 0 ;
105 return accumulatedCompressed + accumulatedOverhead;
111 accumulatedEntropy = 0;
112 accumulatedOverhead = 0;
113 accumulatedCompressed = 0;
114 accumulatedUncompressed = 0;
121 return accumulatedUncompressed;
126 accumulatedUncompressed = 0 ;
133 if( accumulatedCompressed + accumulatedOverhead > 0 )
134 return accumulatedUncompressed / (accumulatedCompressed + accumulatedOverhead );
144 if( ps != NULL )
delete ps;
150 typename Grid::GlobalIdSet
const& idSet = grid.globalIdSet();
153 for(
int level = grid.maxLevel(); level >= 0; --level )
155 auto lEnd = grid.levelGridView(level).template end<dim>();
156 for(
auto it = grid.levelGridView(level).template begin<dim>(); it != lEnd; ++it )
158 levelInfo[idSet.id( *it )] = level ;
170 void encode( Grid
const& grid, CoeffVector
const& sol, std::string fn,
double aTol_ = 0,
int maxLevel_ = -1 )
172 std::ofstream out( fn.c_str(), std::ios::binary ) ;
173 encode( grid, sol, out, aTol_, maxLevel_);
194 void encode( Grid
const& grid, CoeffVector
const& sol, std::ostream& out,
double aTol_,
int maxLevel_ = -1
197 double aTolSave = aTol ;
198 if( aTol_ > 0 ) aTol = aTol_ ;
201 int maxLevel = maxLevel_;
204 maxLevel = grid.maxLevel() ;
207 double overhead = 0, entropy = 0, compressed = 0 ;
209 size_t nNodes = grid.size(maxLevel,
dim);
213 typename Grid::GlobalIdSet
const& idSet = grid.globalIdSet();
217 std::vector<long int> indices ;
218 std::vector<double> predictionError ;
220 std::vector<std::vector<long int> > levelIndices;
226 for(
size_t i = 0; i < grid.size(coarseLevel,
dim); i++ )
230 predictionError.push_back( -sol[ ps->getIndexOnLevel(coarseLevel, i, maxLevel) ] );
234 quantize( predictionError, indices) ;
235 reconstruct( predictionError, indices ) ;
237 levelIndices.push_back(indices);
239 CoeffVector reconstruction(grid.size(coarseLevel,
dim) ) ;
242 size_t nEntries = sol.size();
243 std::vector<long int> intervalIndicesTmp( nEntries );
244 long int minIndex = 0;
246 size_t vertexCount = 0 ;
248 for(
size_t i = 0; i < grid.size(coarseLevel,
dim); i++ )
250 reconstruction[vertexCount] = -predictionError[vertexCount] ;
252 long int tmpIndex = indices[vertexCount];
253 intervalIndicesTmp[ps->getIndexOnLevel(coarseLevel, i, maxLevel)] = tmpIndex;
254 if( tmpIndex < minIndex ) minIndex = tmpIndex;
259 CoeffVector prediction;
261 for(
int l = coarseLevel ; l < maxLevel ; l++ )
263 ps->mv(l, reconstruction, prediction) ;
266 std::vector<std::vector<size_t> > vertexInfo;
267 std::vector<size_t> idxVec(3);
270 predictionError.clear();
271 predictionError.reserve( prediction.size() );
274 typename Grid::LevelGridView::IndexSet
const& levelIndexSet = grid.levelGridView(l+1).indexSet();
276 auto itEnd = grid.levelGridView(l+1).template end<dim>();
277 for(
auto it = grid.levelGridView(l+1).template begin<dim>(); it != itEnd; ++it)
279 unsigned char vertexLevel = levelInfo[idSet.id( *it )] ;
280 auto levelIdx = levelIndexSet.index(*it);
281 if( vertexLevel == l+1 )
287 auto solIdx = ps->getIndexOnLevel(l+1, levelIdx, maxLevel);
288 predictionError.push_back(prediction[levelIdx] - sol[solIdx] );
290 idxVec[0] = levelIdx;
292 idxVec[2] = vertexLevel;
293 vertexInfo.push_back(idxVec);
305 quantize( predictionError, indices) ;
306 levelIndices.push_back(indices);
308 if( (l+1) < maxLevel )
310 reconstruct( predictionError, indices) ;
314 reconstruction = prediction;
317 for(
size_t ii = 0 ; ii < vertexInfo.size(); ii++ )
320 unsigned char vertexLevel = vertexInfo[ii][2];
321 IndexType levelIndex = vertexInfo[ii][0];
322 if( vertexLevel < l+1 )
330 long int tmpIndex = indices[vertexCount];
331 intervalIndicesTmp[vertexInfo[ii][1] ] = tmpIndex;
332 if( tmpIndex < minIndex ) minIndex = tmpIndex;
334 reconstruction[levelIndex] -= predictionError[vertexCount] ;
342 for(
size_t ii = 0 ; ii < vertexInfo.size(); ii++ )
344 unsigned char vertexLevel = vertexInfo[ii][2];
345 if( vertexLevel < l+1 ) {
continue; }
347 long int tmpIndex = indices[vertexCount];
348 intervalIndicesTmp[vertexInfo[ii][1] ] = tmpIndex;
349 if( tmpIndex < minIndex ) minIndex = tmpIndex;
356 std::vector<ULong> intervalIndices( nEntries ) ;
357 for(
size_t i = 0; i < nEntries; i++ ) intervalIndices[i] = intervalIndicesTmp[i] - minIndex ;
359 out.write(
reinterpret_cast<char*
>( &minIndex ),
sizeof(
long int) ) ;
363 std::unordered_map<unsigned long, unsigned long> freqMap;
364 unsigned int nnz = 0;
365 for(
size_t i = 0 ; i < nEntries ; i++ )
367 if( freqMap.find(intervalIndices[i] ) != freqMap.end() )
369 freqMap[intervalIndices[i]]++ ;
374 freqMap[intervalIndices[i]] = 1;
378 out.write(
reinterpret_cast<char*
>( &nnz ),
sizeof(
unsigned int) ) ;
383 std::vector<unsigned long> frequenciesForRangeCoder(nnz);
386 std::unordered_map<unsigned long, unsigned long> dictMap;
387 unsigned long curNo = 0 ;
389 std::unordered_map<unsigned long,unsigned long>::iterator mapIt;
390 std::unordered_map<unsigned long,unsigned long>::iterator mapEnd = freqMap.end();
391 for( mapIt = freqMap.begin(); mapIt != mapEnd; ++mapIt )
393 unsigned long lv = mapIt->first;
395 frequenciesForRangeCoder[curNo] = mapIt->second;
397 dictMap.insert( dictMap.begin(), std::pair<unsigned long, unsigned long>(lv, curNo) ) ;
398 out.write(
reinterpret_cast<char*
>( &lv ),
sizeof(
unsigned long) ) ;
404 for(
unsigned int i = 0 ; i < nnz ; i++ )
407 out.write(
reinterpret_cast<char*
>( &frequenciesForRangeCoder[i] ),
sizeof(
unsigned long) ) ;
429 for(
size_t i = 0 ; i < nEntries; i++ )
431 encodeSymbol( encoder, alphabet, dictMap.find(intervalIndices[i])->second );
437 size_t symbolCounter = 0 ;
438 std::vector<unsigned long> count(nnz, 1) ;
443 for(
size_t i = 0 ; i < nEntries; i++ )
445 encodeSymbol( encoder, alphabet, dictMap.at(intervalIndices[i]));
446 ++
count[dictMap[intervalIndices[i]]];
448 if (symbolCounter>0.1*nEntries)
467 void decode( Grid
const& gridState, CoeffVector& sol, std::string fn,
double aTol_ = 0,
int maxLevel_ = -1 )
469 std::ifstream in( fn.c_str(), std::ios::binary ) ;
470 decode(gridState, sol, in, aTol_, maxLevel_);
490 void decode( Grid
const& gridState, CoeffVector& sol, std::istream& in,
double aTol_,
int maxLevel_ = -1
493 double aTolSave = aTol ;
494 if( aTol_ > 0 ) aTol = aTol_ ;
496 int maxLevel = maxLevel_;
497 if( maxLevel < 0 ) maxLevel = gridState.maxLevel();
500 typename Grid::GlobalIdSet
const& idSet = gridState.globalIdSet();
503 std::vector<double> values ;
504 std::vector<long int> intervalIndices( gridState.size(maxLevel,
dim));
505 size_t nEntries = intervalIndices.size();
511 in.read(
reinterpret_cast<char*
>( &minIndex ),
sizeof(
long int) ) ;
515 in.read(
reinterpret_cast<char*
>( &nnz ),
sizeof(
unsigned int) ) ;
518 std::vector<long int> dictionary( nnz ) ;
519 for(
int i = 0 ; i < nnz ; i++ )
521 in.read(
reinterpret_cast<char*
>( &dictionary[i] ),
sizeof(
unsigned long) ) ;
524 std::vector<unsigned long> frequencies( nnz, 0 ) ;
525 for(
int i = 0 ; i < nnz ; i++ )
527 in.read(
reinterpret_cast<char*
>( &frequencies[i] ),
sizeof(
unsigned long) ) ;
534 for(
int i = 0 ; i < intervalIndices.size() ; i++ )
536 unsigned long s = decodeSymbol(decoder,alphabet) ;
537 intervalIndices[i] = dictionary[s] + minIndex ;
540 catch( std::ios_base::failure& e )
542 if (in.rdstate() & std::ifstream::eofbit)
544 std::cout <<
"EOF reached.\n";
548 std::cout <<
" Decoding error\n" << e.what() <<
"\n";
552 size_t symbolCounter = 0 ;
554 std::vector<unsigned long> symcount(nnz, 1) ;
560 for(
size_t i = 0 ; i < nEntries ; i++ )
562 unsigned long s = decodeSymbol(decoder,alphabet) ;
563 intervalIndices[i] = dictionary[s] + minIndex ;
568 if (symbolCounter>0.1*nEntries)
570 alphabet.
update(symcount.begin(),symcount.end());
575 catch( std::ios_base::failure& e )
577 if (in.rdstate() & std::ifstream::eofbit)
579 std::cout <<
"EOF reached.\n";
583 std::cout <<
" Decoding error\n" << e.what() <<
"\n";
593 CoeffVector reconstruction( gridState.size(coarseLevel,
dim) );
594 sol.resize( gridState.size(maxLevel,
dim) );
598 std::vector<double> reconstructedValues;
599 reconstruct(reconstructedValues, intervalIndices);
603 for(
size_t i = 0; i < gridState.size(coarseLevel,
dim); i++ )
606 auto solIdx = ps->getIndexOnLevel(coarseLevel, i, maxLevel);
608 double recVal = reconstructedValues[solIdx];
609 reconstruction[i] = -recVal ;
612 sol[ solIdx] = -recVal ;
619 CoeffVector prediction;
621 for(
int l = coarseLevel ; l < maxLevel ; l++ )
623 ps->mv( l, reconstruction, prediction ) ;
628 reconstruction = prediction;
630 typename Grid::LevelGridView::IndexSet
const& levelIndexSet = gridState.levelGridView(l+1).indexSet();
634 auto itEnd = gridState.levelGridView(l+1).template end<dim>();
635 for(
auto it = gridState.levelGridView(l+1).template begin<dim>(); it != itEnd ; ++it)
637 IndexType levelIndex = levelIndexSet.index(*it);
638 auto solIdx = ps->getIndexOnLevel(l+1, levelIndex, maxLevel);
639 if(levelInfo[idSet.id(*it)] == l+1)
643 reconstruction[levelIndex] -= reconstructedValues[solIdx];
650 sol[solIdx] = reconstruction[ levelIndex] ;
669 void quantize( std::vector<double>
const& values, std::vector<long int>& indices)
672 indices.resize( values.size(), 0 ) ;
674 for(
size_t i = 0 ; i < values.size() ; i++ )
676 indices[i] =
static_cast<long int>( floor( values[i] / (2*aTol) + 0.5 ) );
682 void reconstruct( std::vector<double>& values, std::vector<long int>
const& indices)
685 values.resize( indices.size() ) ;
686 for(
size_t i = 0 ; i < indices.size() ; i++ )
688 values[i] = indices[i] * 2* aTol ;
695 double reconstruct(
long int const& index )
701 double ld(
double val ) {
return log(val)/log(2.0) ; }
704 std::unordered_map<IdType, unsigned char> levelInfo ;
707 double accumulatedEntropy, accumulatedOverhead, accumulatedUncompressed, accumulatedCompressed;
A simple alphabet of symbols with frequencies to be used with the range coder.
void update(InIter first, InIter last)
Modifies the symbols' half-open ranges to represent the symbols' frequencies given in the range [firs...
void setup(Grid const &grid)
Grid::template Codim< dim >::LevelIterator VertexLevelIterator
void encode(Grid const &grid, CoeffVector const &sol, std::ostream &out, double aTol_, int maxLevel_=-1)
Grid::template Codim< dim >::LeafIterator VertexLeafIterator
double reportOverallSize()
void decode(Grid const &gridState, CoeffVector &sol, std::string fn, double aTol_=0, int maxLevel_=-1)
Grid::Traits::GlobalIdSet::IdType IdType
double reportUncompressedSize()
Grid::LeafGridView::IndexSet::IndexType LeafIndexType
void encode(Grid const &grid, CoeffVector const &sol, std::string fn, double aTol_=0, int maxLevel_=-1)
Grid::LevelGridView::IndexSet::IndexType IndexType
void decode(GridManager< Grid > &gridManagerState, typename VariableSet::VariableSet &sol, std::string fn)
LossyStorage(int coarseLevel_, double aTol_)
void resetUncompressedSize()
VariableSet::VariableSet encode(typename VariableSet::VariableSet const &sol, std::string fn)
LossyStorage(GridManager< Grid > &gridManager_, VariableSet const &varSet_, int coarseLevel_, double aTol_, bool uniform_=false, QuantizationPolicy &quantizationPolicy_=QuantizationPolicy())
Grid::LeafIndexSet IndexSet
double reportCompressedSize()
void resetCompressedSize()
void decode(Grid const &gridState, CoeffVector &sol, std::istream &in, double aTol_, int maxLevel_=-1)
Entropy coding with range decoder.
Entropy coding with range encoder.
int count(Cell const &cell, int codim)
Computes the number of subentities of codimension c of a given entity.
Range coder for fast entropy coding.