56 if sys.version[0] ==
'2':
63 from daal.data_management
import Compressor_Zlib, Decompressor_Zlib
65 utils_folder = os.path.realpath(os.path.abspath(os.path.dirname(os.path.dirname(__file__))))
66 if utils_folder
not in sys.path:
67 sys.path.insert(0, utils_folder)
68 from utils
import getCRC32, readTextFile
70 datasetFileName = os.path.join(
'..',
'data',
'batch',
'logitboost_train.csv')
73 sendReceiveQueue = Queue.Queue()
75 maxDataBlockSize = 16384
77 def getUncompressedDataBlock(sentDataStream, availableDataSize):
78 cur_pos = sentDataStream.size - availableDataSize
81 if availableDataSize >= maxDataBlockSize:
82 return (sentDataStream[cur_pos:cur_pos + maxDataBlockSize], availableDataSize - maxDataBlockSize)
83 elif availableDataSize < maxDataBlockSize
and availableDataSize > 0:
84 return (sentDataStream[cur_pos:cur_pos + availableDataSize], 0)
88 def sendCompressedDataBlock(block):
89 currentBlock = np.copy(block)
91 sendReceiveQueue.put(currentBlock)
94 def receiveCompressedDataBlock():
96 if sendReceiveQueue.empty():
99 return np.copy(sendReceiveQueue.get())
102 def printCRC32(sentDataStream, receivedDataStream):
104 crcSentDataStream = getCRC32(sentDataStream)
105 crcReceivedDataStream = getCRC32(receivedDataStream)
107 print(
"\nCompression example program results:\n")
109 print(
"Input data checksum: 0x{:02X}".format(crcSentDataStream))
110 print(
"Received data checksum: 0x{:02X}".format(crcReceivedDataStream))
112 if sentDataStream.size != receivedDataStream.size:
113 print(
"ERROR: Received data size mismatches with the sent data size")
115 elif crcSentDataStream != crcReceivedDataStream:
116 print(
"ERROR: Received data CRC mismatches with the sent data CRC")
118 print(
"OK: Received data CRC matches with the sent data CRC")
121 if __name__ ==
"__main__":
124 sentDataStream = readTextFile(datasetFileName)
127 compressedDataBlock = np.empty(maxDataBlockSize, dtype=np.uint8)
128 receivedDataStream = np.empty(sentDataStream.size, dtype=np.uint8)
131 compressor = Compressor_Zlib()
134 (uncompressedDataBlock, availableDataSize) = getUncompressedDataBlock(sentDataStream, sentDataStream.size)
135 while uncompressedDataBlock
is not None:
137 compressor.setInputDataBlock(uncompressedDataBlock, 0)
142 compressor.run(compressedDataBlock, 0)
145 compressedDataView = compressedDataBlock[0:compressor.getUsedOutputDataBlockSize()]
148 sendCompressedDataBlock(compressedDataView)
151 if not compressor.isOutputDataBlockFull():
155 (uncompressedDataBlock, availableDataSize) = getUncompressedDataBlock(sentDataStream, availableDataSize)
158 decompressor = Decompressor_Zlib()
161 compressedDataBlock = receiveCompressedDataBlock()
164 while compressedDataBlock
is not None:
166 decompressor.setInputDataBlock(compressedDataBlock, 0)
169 decompressor.run(receivedDataStream, offset)
172 offset += decompressor.getUsedOutputDataBlockSize()
175 compressedDataBlock = receiveCompressedDataBlock()
178 printCRC32(sentDataStream, receivedDataStream)