StorageTransformations.hpp

#ifndef __FIBER_BUNDLE_STORAGETRANSFORMATIONS_HPP
#define __FIBER_BUNDLE_STORAGETRANSFORMATIONS_HPP
 
#include <memcore/Interface.hpp>
#include <memcore/typemap.hpp>
 
 
#include "BundleAPI.h"
#include "GridID.hpp"
 
namespace       Fiber
{
 
class   BUNDLE_API StorageTransformations : public MemCore::Interface<StorageTransformations>
                                          , public MemCore::typemap<MemCore::TypeInfo>
{
public:
        class   BUNDLE_API IOPerformance : public ReferenceBase<IOPerformance>
        {
        public:
                IOPerformance()
                : ReferenceBase<IOPerformance>(this)
                {}
 
                ~IOPerformance();
 
                virtual void    beginWrite(
                        const MemCore::RefPtr<MemCore::ChunkBase>&Cb,
                        const MemCore::RefPtr<Fiber::FragmentID>&fID,
                        const std::string&      theFieldName,
                        const std::string&      theRepresenterName,
                        const Fiber::SkeletonID&theSkeletonID,
                        const std::string&      theGridname,
                        double                  theTime
                        ) = 0;
 
                virtual void    endWrite(
                        double  SecondsUsedToWriteData,
                        const MemCore::RefPtr<MemCore::ChunkBase>&Mb,
                        const MemCore::RefPtr<Fiber::FragmentID>&fID,
                        const std::string&      theFieldName,
                        const std::string&      theRepresenterName,
                        const Fiber::SkeletonID&theSkeletonID,
                        const std::string&      theGridname,
                        double                  theTime
                        ) = 0;
 
                virtual void    doneRead(
                        double  SecondsUsedToReadData,
                        size_t  BytesRead,
                        CreativeArrayBase&theCreator,
                        const DynamicSize&DS,
                        int     NumberOfAvailableFragments,
                        int     CurrentFragmentNumber);
 
        };
 
        RefPtr<IOPerformance>   myIOPerformance;
 
 
        /*
          TODO: The following compression variables should go in a struct
          that is returned by a function that is called with the full context information.
         */
 
 
        int     CompressViaLZ4Filter = -1;
 
        int     GzipCompressionLevel = -1;
 
        bool    DoScaling = false;
 
        int     zstdCompressionLevel = -1;
        int     zstdDictionary = -1;
 
 
        bool    EnableShuffleFilter = false;
 
        uint32_t ScaleFactor = 0;
 
 
        int                             HDF5_CustomFilter = 0;
        std::vector<unsigned int>       HDF5FilterParameters;
 
 
        bool    useWindowsNativeVFDforHDF5 = true;
 
 
        int     Power10ScaleFactor = 0;
 
        virtual bool    SkipStorage( double time, const string&gridname,
                                     const MemCore::WeakPtr<Fiber::Grid>&myGrid,
                                     const SkeletonID&Sid,
                                     const string&RepresenterName,
                                     const Field&F,
                                     const string&FieldName,
                                     Fiber::CreativeArrayBase&theCreator,
                                     const MemCore::RefPtr<Fiber::FragmentID>&fID
                                    ) const;
 
 
        struct  BUNDLE_API      PrecisionTransformation
        {
                std::set<string>        Fieldnames_to_AlwaysWrite_as_SinglePrecision_if_possible,
                                        Fieldnames_to_NeverWrite_as_SinglePrecision;
 
                bool    doEnableWriteSinglePrecisionForCoordinates = false,
                        doEnableWriteSinglePrecisionForVectors     = false,
                        doEnableWriteSinglePrecisionForBivectors   = false;
 
                bool    WriteAsSinglePrecision(const std::string&FieldName, const std::type_info&theType) const;
 
                bool    EnableNumericalShift(const std::string&FieldName, const std::type_info&theType) const;
 
                void    enable(bool toWhat = true)
                {
                        doEnableWriteSinglePrecisionForCoordinates =
                        doEnableWriteSinglePrecisionForVectors     =
                        doEnableWriteSinglePrecisionForBivectors   = toWhat;
                }
 
                void    disable()
                {
                        enable(false);
                }
        };
 
 
        struct  BUNDLE_API      FilterSettings
        {
                enum class SzipCoding
                {
                        None,
                        Entropy,
                        NearestNeighbor
                };
 
                SzipCoding SzipCodingMethod = SzipCoding::None;
 
                unsigned int szip_pixels_per_block = 16;
 
 
                enum    Compressor
                {
                None = -1,
                BLOSCLZ   = 0, 
                LZ4       = 1,
                LZ4HC     = 2,
                SNAPPY    = 3,
                ZLIB      = 4,
                ZSTD      = 5
                };
 
                Compressor      BLOSC = None;
 
                enum    Shuffle
                {
                NoShuffle    = -1,
                DOSHUFFLE    = 1, /* byte-wise shuffle */
                MEMCPYED     = 0, /* plain copy */
                DOBITSHUFFLE = 2  /* bit-wise shuffle */
                };
 
                Shuffle ShuffleMode = NoShuffle;
 
                int     CompressionLevel = 0;
 
                bool    ComputeRangeAttributeOnWriting = false;
 
                struct  ZFP
                {
                        enum
                        {
                                none,
                                rate,
                                precision,
                                accuracy,
                                expert,
                                lossless,
                                zfp_default
 
                        }       mode = none;
 
                        double          data_rate,
                                        data_accuracy;
 
                        unsigned        data_precision;
 
                        unsigned        minbits, maxbits, maxprec,  minexp;
 
                }
                        zfp;
 
                enum    AutoSettingStrategy
                {
                        Uncompressed,
                        BestForReading,
                        BestForWriting,
                        SmallestSize
                };
 
                bool    doEnableWriteSinglePrecision =  false,
                        doEnableNumericalShift = false;
 
                void    setAutomaticStrategy(AutoSettingStrategy StrategyMode = BestForReading);
 
                FilterSettings()
                {}
 
                FilterSettings(AutoSettingStrategy StrategyMode)
                {
                        setAutomaticStrategy(StrategyMode);
                }
 
                string  human_readable_description() const;
        };
 
//      FilterSettings::AutoSettingStrategy StrategyMode = FilterSettings::BestForReading;
        FilterSettings::AutoSettingStrategy StrategyMode = FilterSettings::Uncompressed;
 
        struct  FilterContext
        {
                double time;
                const string gridname;
                const MemCore::WeakPtr<Fiber::Grid> myGrid;
                const SkeletonID Sid;
                const string RepresenterName;
                const MemCore::RefPtr<Field> F;
                const string FieldName;
                MemCore::RefPtr<Fiber::CreativeArrayBase> theCreator;
                const MemCore::RefPtr<Fiber::FragmentID> fID;
                const MemCore::RefPtr<MemBase>&Mb;
        };
 
 
        PrecisionTransformation defaultPrecisionTransformation;
 
        virtual FilterSettings  getFilterSettings( double time, const string&gridname,
                                                   const MemCore::WeakPtr<Fiber::Grid>&myGrid,
                                                   const SkeletonID&Sid,
                                                   const string&RepresenterName,
                                                   const Field&F,
                                                   const string&FieldName,
                                                   Fiber::CreativeArrayBase&theCreator,
                                                   const MemCore::RefPtr<Fiber::FragmentID>&fID,
                                                   const MemCore::RefPtr<MemBase>&Mb) const;
 
 
        bool    StoreFragmentsSeparately = false;
        bool    StoreFieldsSeparately = false;
        bool    StoreGridsSeparately = false;
 
        bool    trulyDeleteFileContents = false;
 
        std::set<RefPtr<FieldID>>       SaveableFields;
 
        std::set<RefPtr<GridID>>        SaveableGrids;
 
 
        StorageTransformations(bool AutomaticSetup = false);
 
        ~StorageTransformations();
 
        void    autoSetup();
 
 
        virtual bool isSaveableTime(double T) const;
 
        struct  HDF5_Tuning
        {
                string  elink_prefix;
 
 
               int      ELINK_CACHESIZE = 128*1024;
 
                int     FileSpaceStrategyThreshold = 4096;
 
                int     sieve_buf_size = 0;
 
                int     MDC_initial_size = 32*1024,
                        MDC_max_size     = 64*1024,
                        MDC_min_size     = 1024;
 
        }
                HDF5;
 
        virtual std::pair<string,string> getExternalLocation( double time, const string&gridname,
                                                              const SkeletonID&Sid,
                                                              const string&RepresenterName,
                                                              const string&FieldName ) const;
 
        virtual string getExternalLocationPrefix( double time, const string&gridname,
                                                  const SkeletonID&Sid,
                                                  const string&RepresenterName,
                                                  const string&FieldName ) const;
 
        virtual void    setFieldPacking( double time, const string&gridname,
                                         const MemCore::WeakPtr<Fiber::Grid>&myGrid,
                                         const SkeletonID&Sid,
                                         const string&RepresenterName,
                                         const Field&F,
                                         const string&FieldName,
                                         Fiber::CreativeArrayBase&theCreator,
                                         const MemCore::RefPtr<Fiber::FragmentID>&fID) const;
 
 
        bool    EnableExternalFileReference = false;
 
        virtual string  getExternalFileReference(const string&MainFile, const string&FragmentFile);
 
 
        bool    AllowExternalLinkingOnWritingOutOfRAMData = false;
 
        virtual std::string
                getExternalLinkFilename(const std::string&link_containing_file_name,
                                        const std::string&external_target_file_name) const;
 
        virtual DynamicSize computeChunkSize(const DynamicSize&FragmentSize,
                                             size_t TypeSize, string&StorageInfo);
 
        virtual bool    isSaveableField(const FieldID&id, const Field&F,
                                        const WeakPtr<Fiber::Grid>&theGrid,
                                        const std::string       &theRepresenterName,
                                        double                  theTime,
                                        const std::string       &theGridname,
                                        const Fiber::SkeletonID&theSkeletonID);
 
        virtual bool    isSaveableFragment(
                const RefPtr<FragmentID>&fid, const RefPtr<CreativeArrayBase>&cab,
                const FieldID&id, const Field&F,
                const WeakPtr<Fiber::Grid>&theGrid,
                const std::string       &theRepresenterName,
                double                  theTime,
                const std::string       &theGridname,
                const Fiber::SkeletonID&theSkeletonID);
 
        virtual void    isSaved(const RefPtr<CreativeArrayBase>&cab,
                                const RefPtr<FragmentID>&fid,
                                const FieldID&id, const Field&F,
                                const WeakPtr<Fiber::Grid>&theGrid,
                                const std::string       &theRepresenterName,
                                double                  theTime,
                                const std::string       &theGridname,
                                const Fiber::SkeletonID&theSkeletonID);
};
 
} // namespace Fiber
 
 
#endif // __FIBER_BUNDLE_STORAGETRANSFORMATIONS_HPP