using System; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Runtime.InteropServices; namespace ZstdCompression { public static class Zstd { private static void Call(UIntPtr x) { Debug.WriteLine(x); if (ZSTD_isError(x) == 0) return; throw new IOException(ZSTD_getErrorName(x)); } public class ZStdOutputStream:Stream { private readonly Stream _outputStream; private readonly bool _leaveOpen; private readonly int _inputBufferSize; private readonly IntPtr _zst; private readonly byte[] _outputBufferArray; private bool _closed; public ZStdOutputStream(Stream outputStream, int level=6, bool leaveOpen=false) { _outputStream = outputStream; _leaveOpen = leaveOpen; _zst = ZSTD_createCStream(); Call(ZSTD_initCStream(_zst, level)); _inputBufferSize = (int) ZSTD_CStreamInSize().ToUInt32(); _outputBuffer.Size = ZSTD_CStreamOutSize(); _outputBufferArray = new byte[(int)_outputBuffer.Size.ToUInt32()]; } public override void Close() { if(_closed) return; Flush(false); Flush(true); _outputStream.Flush(); Call(ZSTD_freeCStream(_zst)); if(!_leaveOpen) _outputStream.Close(); _closed = true; base.Close(); } public override void Flush() { Flush(false); _outputStream.Flush(); } public override long Seek(long offset, SeekOrigin origin) { throw new NotSupportedException(); } public override void SetLength(long value) { throw new NotSupportedException(); } public override int Read(byte[] buffer, int offset, int count) { throw new NotSupportedException(); } private void Flush(bool end) { var alloc2 = GCHandle.Alloc(_outputBufferArray, GCHandleType.Pinned); try { _outputBuffer.Data = Marshal.UnsafeAddrOfPinnedArrayElement(_outputBufferArray, 0); _outputBuffer.Position = UIntPtr.Zero; Call(end ? ZSTD_endStream(_zst, _outputBuffer) : ZSTD_flushStream(_zst, _outputBuffer)); _outputStream.Write(_outputBufferArray, 0, (int)_outputBuffer.Position.ToUInt32()); } finally { alloc2.Free(); } } private readonly Buffer _inputBuffer = new Buffer(); private readonly Buffer _outputBuffer = new Buffer(); public override void Write(byte[] buffer, int offset, int count) { var alloc1 = GCHandle.Alloc(buffer, GCHandleType.Pinned); var alloc2 = GCHandle.Alloc(_outputBufferArray, GCHandleType.Pinned); try { _outputBuffer.Data = Marshal.UnsafeAddrOfPinnedArrayElement(_outputBufferArray, 0); _outputBuffer.Position=UIntPtr.Zero; while (count > 0) { var size = Math.Min(count, _inputBufferSize); _outputBuffer.Position = UIntPtr.Zero; _inputBuffer.Data = Marshal.UnsafeAddrOfPinnedArrayElement(buffer, offset); _inputBuffer.Position = UIntPtr.Zero; _inputBuffer.Size = new UIntPtr((uint) size); Call(ZSTD_compressStream(_zst, _outputBuffer, _inputBuffer)); size = (int) _inputBuffer.Position.ToUInt32(); _outputStream.Write(_outputBufferArray, 0, (int) _outputBuffer.Position.ToUInt32()); count -= size; offset += size; } } finally { alloc1.Free(); alloc2.Free(); } } public override bool CanRead { get { return false; } } public override bool CanSeek { get { return false; } } public override bool CanWrite { get { return _outputStream.CanWrite; } } public override long Length { get { return 0; } } public override long Position { get { return 0; } set { throw new NotSupportedException(); } } } public class ZStdInputStream:Stream { private readonly Stream _inputStream; private readonly bool _leaveOpen; private readonly IntPtr _zst; private readonly byte[] _inputBufferArray; public ZStdInputStream(Stream inputStream, bool leaveOpen=false) { _inputStream = inputStream; _leaveOpen = leaveOpen; _zst = ZSTD_createDStream(); Call(ZSTD_initDStream(_zst)); _inputBuffer.Size = ZSTD_DStreamInSize(); _inputBufferArray = new byte[(int) _inputBuffer.Size.ToUInt32()]; _outputBuffer.Size = ZSTD_DStreamOutSize(); } private bool _closed; public override void Close() { if(_closed) return; Call(ZSTD_freeDStream(_zst)); if(!_leaveOpen) _inputStream.Close(); _closed = true; base.Close(); } public override void Flush() { } public override long Seek(long offset, SeekOrigin origin) { throw new NotSupportedException(); } public override void SetLength(long value) { throw new NotSupportedException(); } private int _inputArrayPosition; private int _inputArraySize; private bool _depleted; public override int Read(byte[] buffer, int offset, int count) { if (count == 0) return 0; var retVal = 0; var alloc1 = GCHandle.Alloc(_inputBufferArray, GCHandleType.Pinned); var alloc2 = GCHandle.Alloc(buffer, GCHandleType.Pinned); try { while (count > 0) { var left = _inputArraySize - _inputArrayPosition; if (left <= 0 && !_depleted) { _inputArrayPosition = 0; _inputArraySize = left = _inputStream.Read(_inputBufferArray, 0, _inputBufferArray.Length); // no more data at all if (left <= 0) { left = 0; _depleted = true; } } _inputBuffer.Position = UIntPtr.Zero; if (_depleted) { _inputBuffer.Size = UIntPtr.Zero; _inputBuffer.Data = IntPtr.Zero; } else { _inputBuffer.Size = new UIntPtr((uint)left); _inputBuffer.Data = Marshal.UnsafeAddrOfPinnedArrayElement(_inputBufferArray, _inputArrayPosition); } _outputBuffer.Position = UIntPtr.Zero; _outputBuffer.Size = new UIntPtr((uint)count); _outputBuffer.Data = Marshal.UnsafeAddrOfPinnedArrayElement(buffer, offset); Call(ZSTD_decompressStream(_zst, _outputBuffer, _inputBuffer)); var bytesProduced = (int)_outputBuffer.Position.ToUInt32(); if(bytesProduced==0 && _depleted) break; retVal += bytesProduced; count -= bytesProduced; offset += bytesProduced; if(_depleted) continue; var bytesConsumed = (int)_inputBuffer.Position.ToUInt32(); _inputArrayPosition += bytesConsumed; } return retVal; } finally { alloc1.Free(); alloc2.Free(); } } private readonly Buffer _inputBuffer = new Buffer(); private readonly Buffer _outputBuffer = new Buffer(); public override void Write(byte[] buffer, int offset, int count) { throw new NotSupportedException(); } public override bool CanRead { get { return _inputStream.CanRead; } } public override bool CanSeek { get { return false; } } public override bool CanWrite { get { return false; } } public override long Length { get { return 0; } } public override long Position { get { return 0; } set { } } } [StructLayout(LayoutKind.Sequential)] [SuppressMessage("ReSharper", "NotAccessedField.Local")] private sealed class Buffer { public IntPtr Data; public UIntPtr Size; public UIntPtr Position; } // https://github.com/facebook/zstd/blob/dev/lib/zstd.h // https://facebook.github.io/zstd/zstd_manual.html // https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md public static bool IsZstdStream(byte[] buffBytes, int buffLen) { //0xFD2FB528 LE return buffLen > 3 && buffBytes[0] == 0x28 && buffBytes[1] == 0xB5 && buffBytes[2] == 0x2F && buffBytes[3] == 0xFD; } private const string DllName = "libzstd"; [DllImport(DllName, EntryPoint = "ZSTD_maxCLevel", CallingConvention = CallingConvention.Cdecl)] public static extern int GetMaxCompessionLevel(); [DllImport(DllName, EntryPoint = "ZSTD_versionNumber", CallingConvention = CallingConvention.Cdecl)] public static extern int GetVersionNumber(); //[DllImport(DllName, EntryPoint = "ZSTD_versionString", CallingConvention = CallingConvention.Cdecl)] //public static extern string ZSTD_versionString(); public static string GetVersionString() { var n = GetVersionNumber(); return string.Format("{0}.{1}.{2}", n/10000, (n%10000)/100, n%100); } [DllImport(DllName, EntryPoint = "ZSTD_isError", CallingConvention = CallingConvention.Cdecl)] private static extern int ZSTD_isError(UIntPtr code); [DllImport(DllName, EntryPoint = "ZSTD_getErrorName", CallingConvention = CallingConvention.Cdecl)] private static extern string ZSTD_getErrorName(UIntPtr code); [DllImport(DllName, EntryPoint = "ZSTD_createCStream", CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr ZSTD_createCStream(); [DllImport(DllName, EntryPoint = "ZSTD_freeCStream", CallingConvention = CallingConvention.Cdecl)] private static extern UIntPtr ZSTD_freeCStream(IntPtr zcs); [DllImport(DllName, EntryPoint = "ZSTD_initCStream", CallingConvention = CallingConvention.Cdecl)] private static extern UIntPtr ZSTD_initCStream(IntPtr zcs, int compressionLevel); [DllImport(DllName, EntryPoint = "ZSTD_compressStream", CallingConvention = CallingConvention.Cdecl)] private static extern UIntPtr ZSTD_compressStream(IntPtr zcs, [MarshalAs(UnmanagedType.LPStruct)] Buffer outputBuffer, [MarshalAs(UnmanagedType.LPStruct)] Buffer inputBuffer); [DllImport(DllName, EntryPoint = "ZSTD_flushStream", CallingConvention = CallingConvention.Cdecl)] private static extern UIntPtr ZSTD_flushStream(IntPtr zcs, [MarshalAs(UnmanagedType.LPStruct)] Buffer outputBuffer); [DllImport(DllName, EntryPoint = "ZSTD_endStream", CallingConvention = CallingConvention.Cdecl)] private static extern UIntPtr ZSTD_endStream(IntPtr zcs, [MarshalAs(UnmanagedType.LPStruct)] Buffer outputBuffer); [DllImport(DllName, EntryPoint = "ZSTD_CStreamInSize", CallingConvention = CallingConvention.Cdecl)] private static extern UIntPtr ZSTD_CStreamInSize(); [DllImport(DllName, EntryPoint = "ZSTD_CStreamOutSize", CallingConvention = CallingConvention.Cdecl)] private static extern UIntPtr ZSTD_CStreamOutSize(); [DllImport(DllName, EntryPoint = "ZSTD_createDStream", CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr ZSTD_createDStream(); [DllImport(DllName, EntryPoint = "ZSTD_freeDStream", CallingConvention = CallingConvention.Cdecl)] private static extern UIntPtr ZSTD_freeDStream(IntPtr zcs); [DllImport(DllName, EntryPoint = "ZSTD_initDStream", CallingConvention = CallingConvention.Cdecl)] private static extern UIntPtr ZSTD_initDStream(IntPtr zcs); [DllImport(DllName, EntryPoint = "ZSTD_decompressStream", CallingConvention = CallingConvention.Cdecl)] private static extern UIntPtr ZSTD_decompressStream(IntPtr zcs, [MarshalAs(UnmanagedType.LPStruct)] Buffer outputBuffer, [MarshalAs(UnmanagedType.LPStruct)] Buffer inputBuffer); [DllImport(DllName, EntryPoint = "ZSTD_DStreamInSize", CallingConvention = CallingConvention.Cdecl)] private static extern UIntPtr ZSTD_DStreamInSize(); [DllImport(DllName, EntryPoint = "ZSTD_CStreamOutSize", CallingConvention = CallingConvention.Cdecl)] private static extern UIntPtr ZSTD_DStreamOutSize(); } }