30 if sys.version[0] ==
'2':
37 from daal.data_management
import Compressor_Zlib, Decompressor_Zlib
39 utils_folder = os.path.realpath(os.path.abspath(os.path.dirname(os.path.dirname(__file__))))
40 if utils_folder
not in sys.path:
41 sys.path.insert(0, utils_folder)
42 from utils
import getCRC32, readTextFile
44 datasetFileName = os.path.join(
'..',
'data',
'batch',
'logitboost_train.csv')
47 sendReceiveQueue = Queue.Queue()
49 maxDataBlockSize = 16384
51 def getUncompressedDataBlock(sentDataStream, availableDataSize):
52 cur_pos = sentDataStream.size - availableDataSize
55 if availableDataSize >= maxDataBlockSize:
56 return (sentDataStream[cur_pos:cur_pos + maxDataBlockSize], availableDataSize - maxDataBlockSize)
57 elif availableDataSize < maxDataBlockSize
and availableDataSize > 0:
58 return (sentDataStream[cur_pos:cur_pos + availableDataSize], 0)
62 def sendCompressedDataBlock(block):
63 currentBlock = np.copy(block)
65 sendReceiveQueue.put(currentBlock)
68 def receiveCompressedDataBlock():
70 if sendReceiveQueue.empty():
73 return np.copy(sendReceiveQueue.get())
76 def printCRC32(sentDataStream, receivedDataStream):
78 crcSentDataStream = getCRC32(sentDataStream)
79 crcReceivedDataStream = getCRC32(receivedDataStream)
81 print(
"\nCompression example program results:\n")
83 print(
"Input data checksum: 0x{:02X}".format(crcSentDataStream))
84 print(
"Received data checksum: 0x{:02X}".format(crcReceivedDataStream))
86 if sentDataStream.size != receivedDataStream.size:
87 print(
"ERROR: Received data size mismatches with the sent data size")
89 elif crcSentDataStream != crcReceivedDataStream:
90 print(
"ERROR: Received data CRC mismatches with the sent data CRC")
92 print(
"OK: Received data CRC matches with the sent data CRC")
95 if __name__ ==
"__main__":
98 sentDataStream = readTextFile(datasetFileName)
101 compressedDataBlock = np.empty(maxDataBlockSize, dtype=np.uint8)
102 receivedDataStream = np.empty(sentDataStream.size, dtype=np.uint8)
105 compressor = Compressor_Zlib()
108 (uncompressedDataBlock, availableDataSize) = getUncompressedDataBlock(sentDataStream, sentDataStream.size)
109 while uncompressedDataBlock
is not None:
111 compressor.setInputDataBlock(uncompressedDataBlock, 0)
116 compressor.run(compressedDataBlock, 0)
119 compressedDataView = compressedDataBlock[0:compressor.getUsedOutputDataBlockSize()]
122 sendCompressedDataBlock(compressedDataView)
125 if not compressor.isOutputDataBlockFull():
129 (uncompressedDataBlock, availableDataSize) = getUncompressedDataBlock(sentDataStream, availableDataSize)
132 decompressor = Decompressor_Zlib()
135 compressedDataBlock = receiveCompressedDataBlock()
138 while compressedDataBlock
is not None:
140 decompressor.setInputDataBlock(compressedDataBlock, 0)
143 decompressor.run(receivedDataStream, offset)
146 offset += decompressor.getUsedOutputDataBlockSize()
149 compressedDataBlock = receiveCompressedDataBlock()
152 printCRC32(sentDataStream, receivedDataStream)