OctreePacketData.h

//
//  OctreePacketData.h
//  libraries/octree/src
//
//  Created by Brad Hefta-Gaub on 11/19/2013.
//  Copyright 2013 High Fidelity, Inc.
//
//  TO DO:
//    *  add stats tracking for number of unique colors and consecutive identical colors.
//          (as research for color dictionaries and RLE)
//
//    *  further testing of compression to determine optimal configuration for performance and compression
//
//    *  improve semantics for "reshuffle" - current approach will work for now and with compression
//       but wouldn't work with RLE because the colors in the levels would get reordered and RLE would need
//       to be recalculated
//
//  Distributed under the Apache License, Version 2.0.
//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
 
#ifndef hifi_OctreePacketData_h
#define hifi_OctreePacketData_h
 
#include <atomic>
 
#include <QByteArray>
#include <QString>
#include <QUuid>
 
#include <SharedUtil.h>
#include <ShapeInfo.h>
#include <NLPacket.h>
#include <udt/PacketHeaders.h>
 
#include "EntityShape.h"
#include "MaterialMappingMode.h"
#include "BillboardMode.h"
#include "RenderLayer.h"
#include "PrimitiveMode.h"
#include "WebInputMode.h"
#include "PulseMode.h"
#include "GizmoType.h"
#include "TextEffect.h"
#include "TextAlignment.h"
#include "TextVerticalAlignment.h"
#include "MirrorMode.h"
#include "TonemappingCurve.h"
#include "AmbientOcclusionTechnique.h"
#include "FadeTiming.h"
#include "Sampler.h"
 
#include "OctreeConstants.h"
#include "OctreeElement.h"
 
using AtomicUIntStat = std::atomic<uintmax_t>;
 
typedef unsigned char OCTREE_PACKET_FLAGS;
typedef uint16_t OCTREE_PACKET_SEQUENCE;
const uint16_t MAX_OCTREE_PACKET_SEQUENCE = 65535;
typedef quint64 OCTREE_PACKET_SENT_TIME;
typedef uint16_t OCTREE_PACKET_INTERNAL_SECTION_SIZE;
const int MAX_OCTREE_PACKET_SIZE = udt::MAX_PACKET_SIZE;
 
const unsigned int OCTREE_PACKET_EXTRA_HEADERS_SIZE = sizeof(OCTREE_PACKET_FLAGS)
                + sizeof(OCTREE_PACKET_SEQUENCE) + sizeof(OCTREE_PACKET_SENT_TIME);
 
const unsigned int MAX_OCTREE_PACKET_DATA_SIZE =
    udt::MAX_PACKET_SIZE - (NLPacket::MAX_PACKET_HEADER_SIZE + OCTREE_PACKET_EXTRA_HEADERS_SIZE);
const unsigned int MAX_OCTREE_UNCOMRESSED_PACKET_SIZE = MAX_OCTREE_PACKET_DATA_SIZE;
 
const unsigned int MINIMUM_ATTEMPT_MORE_PACKING = sizeof(OCTREE_PACKET_INTERNAL_SECTION_SIZE) + 40;
const unsigned int COMPRESS_PADDING = 15;
const int REASONABLE_NUMBER_OF_PACKING_ATTEMPTS = 5;
 
const int PACKET_IS_COLOR_BIT = 0;
const int PACKET_IS_COMPRESSED_BIT = 1;
 
class LevelDetails {
    LevelDetails(int startIndex, int bytesOfOctalCodes, int bytesOfBitmasks, int bytesOfColor, int bytesReservedAtStart) :
        _startIndex(startIndex),
        _bytesOfOctalCodes(bytesOfOctalCodes),
        _bytesOfBitmasks(bytesOfBitmasks),
        _bytesOfColor(bytesOfColor),
        _bytesReservedAtStart(bytesReservedAtStart) {
    }
    
    friend class OctreePacketData;
 
private:
    int _startIndex;
    int _bytesOfOctalCodes;
    int _bytesOfBitmasks;
    int _bytesOfColor;
    int _bytesReservedAtStart;
};
 
class OctreePacketData {
public:
    OctreePacketData(bool enableCompression = false, int maxFinalizedSize = MAX_OCTREE_PACKET_DATA_SIZE);
    ~OctreePacketData();
 
    void changeSettings(bool enableCompression = false, unsigned int targetSize = MAX_OCTREE_PACKET_DATA_SIZE);
 
    void reset();
    
    bool startSubTree(const unsigned char* octcode = NULL);
    
    // call to indicate that the current subtree is complete and changes should be committed.
    void endSubTree();
 
    // call rollback the current subtree and restore the stream to the state prior to starting the subtree encoding
    void discardSubTree();
 
    LevelDetails startLevel();
 
    void discardLevel(LevelDetails key);
    
    bool endLevel(LevelDetails key);
 
    bool appendBitMask(unsigned char bitmask);
 
    bool updatePriorBitMask(int offset, unsigned char bitmask); 
 
    bool reserveBitMask();
 
    bool reserveBytes(int numberOfBytes);
 
    bool releaseReservedBitMask();
 
    bool releaseReservedBytes(int numberOfBytes);
 
    bool updatePriorBytes(int offset, const unsigned char* replacementBytes, int length);
 
    bool appendColor(colorPart red, colorPart green, colorPart blue);
 
    bool appendValue(const nodeColor& color);
 
    bool appendValue(uint8_t value);
 
    bool appendValue(uint16_t value);
 
    bool appendValue(uint32_t value);
 
    bool appendValue(quint64 value);
 
    bool appendValue(float value);
 
    bool appendValue(const glm::vec2& value);
 
    bool appendValue(const glm::vec3& value);
 
    bool appendValue(const glm::u8vec3& value);
 
    bool appendValue(const QVector<glm::vec3>& value);
 
    bool appendValue(const QVector<glm::quat>& value);
 
    bool appendValue(const QVector<float>& value);
 
    bool appendValue(const QVector<bool>& value);
 
    bool appendValue(const QVector<QUuid>& value);
 
    bool appendValue(const QSet<QString>& value);
 
    bool appendValue(const glm::quat& value);
 
    bool appendValue(bool value);
 
    bool appendValue(const QString& string);
 
    bool appendValue(const QUuid& uuid);
 
    bool appendValue(const QByteArray& bytes);
 
    bool appendValue(const AACube& aaCube);
 
    bool appendValue(const QRect& rect);
 
    bool appendValue(const Sampler& sampler);
 
    bool appendPosition(const glm::vec3& value);
 
    bool appendRawData(const unsigned char* data, int length);
    bool appendRawData(QByteArray data);
 
    int getUncompressedByteOffset(int offsetFromEnd = 0) const { return _bytesInUse - offsetFromEnd; }
 
    const unsigned char* getFinalizedData();
    int getFinalizedSize();
 
    const unsigned char* getUncompressedData(int byteOffset = 0) { return &_uncompressed[byteOffset]; }
 
    int getUncompressedSize() { return _bytesInUse; }
 
    void setUncompressedSize(int newSize) { _bytesInUse = newSize; }
 
    bool hasContent() const { return (_bytesInUse > 0); }
 
    void loadFinalizedContent(const unsigned char* data, int length);
    
    bool isCompressed() const { return _enableCompression; }
    
    unsigned int getTargetSize() const { return _targetSize; }
 
    int getReservedBytes() { return _bytesReserved; }
 
    int getBytesAvailable() { return _bytesAvailable; }
 
    void debugContent();
    void debugBytes();
    
    static quint64 getCompressContentTime() { return _compressContentTime; } 
    static quint64 getCompressContentCalls() { return _compressContentCalls; } 
    static quint64 getTotalBytesOfOctalCodes() { return _totalBytesOfOctalCodes; }  
    static quint64 getTotalBytesOfBitMasks() { return _totalBytesOfBitMasks; }  
    static quint64 getTotalBytesOfColor() { return _totalBytesOfColor; } 
    
    static int unpackDataFromBytes(const unsigned char* dataBytes, float& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, bool& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, quint64& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, uint32_t& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, uint16_t& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, uint8_t& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, glm::quat& result) { int bytes = unpackOrientationQuatFromBytes(dataBytes, result); return bytes; }
    static int unpackDataFromBytes(const unsigned char* dataBytes, EntityShape& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, ShapeType& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, MaterialMappingMode& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, BillboardMode& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, RenderLayer& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, PrimitiveMode& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, WebInputMode& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, PulseMode& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, GizmoType& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, TextEffect& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, TextAlignment& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, TextVerticalAlignment& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, MirrorMode& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, TonemappingCurve& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, AmbientOcclusionTechnique& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, FadeTiming& result) { memcpy(&result, dataBytes, sizeof(result)); return sizeof(result); }
    static int unpackDataFromBytes(const unsigned char* dataBytes, glm::vec2& result);
    static int unpackDataFromBytes(const unsigned char* dataBytes, glm::vec3& result);
    static int unpackDataFromBytes(const unsigned char* dataBytes, glm::u8vec3& result);
    static int unpackDataFromBytes(const unsigned char* dataBytes, QString& result);
    static int unpackDataFromBytes(const unsigned char* dataBytes, QUuid& result);
    static int unpackDataFromBytes(const unsigned char* dataBytes, QVector<glm::vec3>& result);
    static int unpackDataFromBytes(const unsigned char* dataBytes, QVector<glm::quat>& result);
    static int unpackDataFromBytes(const unsigned char* dataBytes, QVector<float>& result);
    static int unpackDataFromBytes(const unsigned char* dataBytes, QVector<bool>& result);
    static int unpackDataFromBytes(const unsigned char* dataBytes, QVector<QUuid>& result);
    static int unpackDataFromBytes(const unsigned char* dataBytes, QSet<QString>& result);
    static int unpackDataFromBytes(const unsigned char* dataBytes, QByteArray& result);
    static int unpackDataFromBytes(const unsigned char* dataBytes, AACube& result);
    static int unpackDataFromBytes(const unsigned char* dataBytes, QRect& result);
    static int unpackDataFromBytes(const unsigned char* dataBytes, Sampler& result);
 
private:
    bool append(const unsigned char* data, int length);
    
    bool append(unsigned char byte);
 
    unsigned int _targetSize;
    bool _enableCompression;
    
    QByteArray _uncompressedByteArray;
    unsigned char* _uncompressed { nullptr };
    int _bytesInUse;
    int _bytesAvailable;
    int _subTreeAt;
    int _bytesReserved;
    int _subTreeBytesReserved; // the number of reserved bytes at start of a subtree
 
    bool compressContent();
    
    QByteArray _compressedByteArray;
    unsigned char* _compressed { nullptr };
    int _compressedBytes;
    int _bytesInUseLastCheck;
    bool _dirty;
 
    // statistics...
    int _bytesOfOctalCodes;
    int _bytesOfBitMasks;
    int _bytesOfColor;
    int _bytesOfValues;
    int _bytesOfPositions;
    int _bytesOfRawData;
 
    int _bytesOfOctalCodesCurrentSubTree;
 
    static bool _debug;
 
    static AtomicUIntStat _compressContentTime;
    static AtomicUIntStat _compressContentCalls;
 
    static AtomicUIntStat _totalBytesOfOctalCodes;
    static AtomicUIntStat _totalBytesOfBitMasks;
    static AtomicUIntStat _totalBytesOfColor;
    static AtomicUIntStat _totalBytesOfValues;
    static AtomicUIntStat _totalBytesOfPositions;
    static AtomicUIntStat _totalBytesOfRawData;
};
 
#endif // hifi_OctreePacketData_h