Repo for the search and displace ingest module that takes odf, docx and pdf and transforms it into .md to be used with search and displace operations
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

523 lines
15 KiB

3 years ago
  1. {*******************************************************}
  2. { }
  3. { Delphi Supplemental Components }
  4. { ZLIB Data Compression Interface Unit }
  5. { }
  6. { Copyright (c) 1997 Borland International }
  7. { Copyright (c) 1998 Jacques Nomssi Nzali }
  8. { }
  9. {*******************************************************}
  10. {
  11. Modified for
  12. Vampyre Imaging Library
  13. by Marek Mauder
  14. http://imaginglib.sourceforge.net
  15. You can choose which pascal zlib implementation will be
  16. used. IMPASZLIB and FPCPASZLIB are translations of zlib
  17. to pascal so they don't need any *.obj files.
  18. The others are interfaces to *.obj files (Windows) or
  19. *.so libraries (Linux).
  20. Default implementation is IMPASZLIB because it can be compiled
  21. by all supported compilers and works on all supported platforms.
  22. I usually use implementation with the fastest decompression
  23. when building release Win32 binaries.
  24. FPCPASZLIB is useful for Lazarus applications. FPC's zlib is linked
  25. to exe by default so there is no need to link additional (and almost identical)
  26. IMPASZLIB.
  27. There is a small speed comparison table of some of the
  28. supported implementations (TGA image 28�311�570 bytes, compression level = 6,
  29. Delphi 9, Win32, Athlon XP 1900).
  30. ZLib version Decompression Compression Comp. Size
  31. IMPASZLIB | 1.1.2 | 824 ms | 4 280 ms | 18 760 133 B
  32. ZLIBEX | 1.2.2 | 710 ms | 1 590 ms* | 19 056 621 B
  33. DELPHIZLIB | 1.0.4 | 976 ms | 9 190 ms | 18 365 562 B
  34. ZLIBPAS | 1.2.3 | 680 ms | 3 790 ms | 18 365 387 B
  35. * obj files are compiled with compression level hardcoded to 1 (fastest)
  36. }
  37. unit dzlib;
  38. {$I ImagingOptions.inc}
  39. interface
  40. {$DEFINE IMPASZLIB}
  41. { $DEFINE ZLIBPAS}
  42. { $DEFINE FPCPASZLIB}
  43. { $DEFINE ZLIBEX}
  44. { $DEFINE DELPHIZLIB}
  45. { Automatically use FPC's PasZLib when compiling with FPC.}
  46. {$IFDEF FPC}
  47. {$UNDEF IMPASZLIB}
  48. {$DEFINE FPCPASZLIB}
  49. {$ENDIF}
  50. uses
  51. {$IF Defined(IMPASZLIB)}
  52. { Use paszlib modified by me for Delphi and FPC }
  53. imzdeflate, imzinflate, impaszlib,
  54. {$ELSEIF Defined(FPCPASZLIB)}
  55. { Use FPC's paszlib }
  56. zbase, paszlib,
  57. {$ELSEIF Defined(ZLIBPAS)}
  58. { Pascal interface to ZLib shipped with ZLib C source }
  59. zlibpas,
  60. {$ELSEIF Defined(ZLIBEX)}
  61. { Use ZlibEx unit }
  62. ZLibEx,
  63. {$ELSEIF Defined(DELPHIZLIB)}
  64. { Use ZLib unit shipped with Delphi }
  65. ZLib,
  66. {$IFEND}
  67. ImagingTypes, SysUtils, Classes;
  68. {$IF Defined(IMPASZLIB) or Defined(FPCPASZLIB) or Defined(ZLIBPAS)}
  69. type
  70. TZStreamRec = z_stream;
  71. {$IFEND}
  72. const
  73. Z_NO_FLUSH = 0;
  74. Z_PARTIAL_FLUSH = 1;
  75. Z_SYNC_FLUSH = 2;
  76. Z_FULL_FLUSH = 3;
  77. Z_FINISH = 4;
  78. Z_OK = 0;
  79. Z_STREAM_END = 1;
  80. Z_NEED_DICT = 2;
  81. Z_ERRNO = -1;
  82. Z_STREAM_ERROR = -2;
  83. Z_DATA_ERROR = -3;
  84. Z_MEM_ERROR = -4;
  85. Z_BUF_ERROR = -5;
  86. Z_VERSION_ERROR = -6;
  87. Z_NO_COMPRESSION = 0;
  88. Z_BEST_SPEED = 1;
  89. Z_BEST_COMPRESSION = 9;
  90. Z_DEFAULT_COMPRESSION = -1;
  91. Z_FILTERED = 1;
  92. Z_HUFFMAN_ONLY = 2;
  93. Z_RLE = 3;
  94. Z_DEFAULT_STRATEGY = 0;
  95. Z_BINARY = 0;
  96. Z_ASCII = 1;
  97. Z_UNKNOWN = 2;
  98. Z_DEFLATED = 8;
  99. type
  100. { Abstract ancestor class }
  101. TCustomZlibStream = class(TStream)
  102. private
  103. FStrm: TStream;
  104. FStrmPos: Integer;
  105. FOnProgress: TNotifyEvent;
  106. FZRec: TZStreamRec;
  107. FBuffer: array [Word] of Byte;
  108. protected
  109. procedure Progress(Sender: TObject); dynamic;
  110. property OnProgress: TNotifyEvent read FOnProgress write FOnProgress;
  111. constructor Create(Strm: TStream);
  112. end;
  113. { TCompressionStream compresses data on the fly as data is written to it, and
  114. stores the compressed data to another stream.
  115. TCompressionStream is write-only and strictly sequential. Reading from the
  116. stream will raise an exception. Using Seek to move the stream pointer
  117. will raise an exception.
  118. Output data is cached internally, written to the output stream only when
  119. the internal output buffer is full. All pending output data is flushed
  120. when the stream is destroyed.
  121. The Position property returns the number of uncompressed bytes of
  122. data that have been written to the stream so far.
  123. CompressionRate returns the on-the-fly percentage by which the original
  124. data has been compressed: (1 - (CompressedBytes / UncompressedBytes)) * 100
  125. If raw data size = 100 and compressed data size = 25, the CompressionRate
  126. is 75%
  127. The OnProgress event is called each time the output buffer is filled and
  128. written to the output stream. This is useful for updating a progress
  129. indicator when you are writing a large chunk of data to the compression
  130. stream in a single call.}
  131. TCompressionLevel = (clNone, clFastest, clDefault, clMax);
  132. TCompressionStream = class(TCustomZlibStream)
  133. private
  134. function GetCompressionRate: Single;
  135. public
  136. constructor Create(CompressionLevel: TCompressionLevel; Dest: TStream);
  137. destructor Destroy; override;
  138. function Read(var Buffer; Count: Longint): Longint; override;
  139. function Write(const Buffer; Count: Longint): Longint; override;
  140. function Seek(Offset: Longint; Origin: Word): Longint; override;
  141. property CompressionRate: Single read GetCompressionRate;
  142. property OnProgress;
  143. end;
  144. { TDecompressionStream decompresses data on the fly as data is read from it.
  145. Compressed data comes from a separate source stream. TDecompressionStream
  146. is read-only and unidirectional; you can seek forward in the stream, but not
  147. backwards. The special case of setting the stream position to zero is
  148. allowed. Seeking forward decompresses data until the requested position in
  149. the uncompressed data has been reached. Seeking backwards, seeking relative
  150. to the end of the stream, requesting the size of the stream, and writing to
  151. the stream will raise an exception.
  152. The Position property returns the number of bytes of uncompressed data that
  153. have been read from the stream so far.
  154. The OnProgress event is called each time the internal input buffer of
  155. compressed data is exhausted and the next block is read from the input stream.
  156. This is useful for updating a progress indicator when you are reading a
  157. large chunk of data from the decompression stream in a single call.}
  158. TDecompressionStream = class(TCustomZlibStream)
  159. public
  160. constructor Create(Source: TStream);
  161. destructor Destroy; override;
  162. function Read(var Buffer; Count: Longint): Longint; override;
  163. function Write(const Buffer; Count: Longint): Longint; override;
  164. function Seek(Offset: Longint; Origin: Word): Longint; override;
  165. property OnProgress;
  166. end;
  167. { CompressBuf compresses data, buffer to buffer, in one call.
  168. In: InBuf = ptr to compressed data
  169. InBytes = number of bytes in InBuf
  170. Out: OutBuf = ptr to newly allocated buffer containing decompressed data
  171. OutBytes = number of bytes in OutBuf }
  172. procedure CompressBuf(const InBuf: Pointer; InBytes: Integer;
  173. var OutBuf: Pointer; var OutBytes: Integer;
  174. CompressLevel: Integer = Z_DEFAULT_COMPRESSION;
  175. CompressStrategy: Integer = Z_DEFAULT_STRATEGY);
  176. { DecompressBuf decompresses data, buffer to buffer, in one call.
  177. In: InBuf = ptr to compressed data
  178. InBytes = number of bytes in InBuf
  179. OutEstimate = zero, or est. size of the decompressed data
  180. Out: OutBuf = ptr to newly allocated buffer containing decompressed data
  181. OutBytes = number of bytes in OutBuf }
  182. procedure DecompressBuf(const InBuf: Pointer; InBytes: Integer;
  183. OutEstimate: Integer; var OutBuf: Pointer; var OutBytes: Integer);
  184. type
  185. EZlibError = class(Exception);
  186. ECompressionError = class(EZlibError);
  187. EDecompressionError = class(EZlibError);
  188. implementation
  189. const
  190. ZErrorMessages: array[0..9] of PAnsiChar = (
  191. 'need dictionary', // Z_NEED_DICT (2)
  192. 'stream end', // Z_STREAM_END (1)
  193. '', // Z_OK (0)
  194. 'file error', // Z_ERRNO (-1)
  195. 'stream error', // Z_STREAM_ERROR (-2)
  196. 'data error', // Z_DATA_ERROR (-3)
  197. 'insufficient memory', // Z_MEM_ERROR (-4)
  198. 'buffer error', // Z_BUF_ERROR (-5)
  199. 'incompatible version', // Z_VERSION_ERROR (-6)
  200. '');
  201. function zlibAllocMem(AppData: Pointer; Items, Size: Cardinal): Pointer;
  202. begin
  203. GetMem(Result, Items*Size);
  204. end;
  205. procedure zlibFreeMem(AppData, Block: Pointer);
  206. begin
  207. FreeMem(Block);
  208. end;
  209. function CCheck(code: Integer): Integer;
  210. begin
  211. Result := code;
  212. if code < 0 then
  213. raise ECompressionError.Create('zlib: ' + ZErrorMessages[2 - code]);
  214. end;
  215. function DCheck(code: Integer): Integer;
  216. begin
  217. Result := code;
  218. if code < 0 then
  219. raise EDecompressionError.Create('zlib: ' + ZErrorMessages[2 - code]);
  220. end;
  221. procedure CompressBuf(const InBuf: Pointer; InBytes: Integer;
  222. var OutBuf: Pointer; var OutBytes: Integer;
  223. CompressLevel, CompressStrategy: Integer);
  224. var
  225. strm: TZStreamRec;
  226. P: Pointer;
  227. begin
  228. FillChar(strm, sizeof(strm), 0);
  229. {$IFNDEF FPCPASZLIB}
  230. strm.zalloc := @zlibAllocMem;
  231. strm.zfree := @zlibFreeMem;
  232. {$ENDIF}
  233. OutBytes := ((InBytes + (InBytes div 10) + 12) + 255) and not 255;
  234. GetMem(OutBuf, OutBytes);
  235. try
  236. strm.next_in := InBuf;
  237. strm.avail_in := InBytes;
  238. strm.next_out := OutBuf;
  239. strm.avail_out := OutBytes;
  240. CCheck(deflateInit2(strm, CompressLevel, Z_DEFLATED, MAX_WBITS,
  241. DEF_MEM_LEVEL, CompressStrategy));
  242. try
  243. while CCheck(deflate(strm, Z_FINISH)) <> Z_STREAM_END do
  244. begin
  245. P := OutBuf;
  246. Inc(OutBytes, 256);
  247. ReallocMem(OutBuf, OutBytes);
  248. strm.next_out := Pointer(PtrUInt(OutBuf) + (PtrUInt(strm.next_out) - PtrUInt(P)));
  249. strm.avail_out := 256;
  250. end;
  251. finally
  252. CCheck(deflateEnd(strm));
  253. end;
  254. ReallocMem(OutBuf, strm.total_out);
  255. OutBytes := strm.total_out;
  256. except
  257. zlibFreeMem(nil, OutBuf);
  258. raise
  259. end;
  260. end;
  261. procedure DecompressBuf(const InBuf: Pointer; InBytes: Integer;
  262. OutEstimate: Integer; var OutBuf: Pointer; var OutBytes: Integer);
  263. var
  264. strm: TZStreamRec;
  265. P: Pointer;
  266. BufInc: Integer;
  267. begin
  268. FillChar(strm, sizeof(strm), 0);
  269. {$IFNDEF FPCPASZLIB}
  270. strm.zalloc := @zlibAllocMem;
  271. strm.zfree := @zlibFreeMem;
  272. {$ENDIF}
  273. BufInc := (InBytes + 255) and not 255;
  274. if OutEstimate = 0 then
  275. OutBytes := BufInc
  276. else
  277. OutBytes := OutEstimate;
  278. GetMem(OutBuf, OutBytes);
  279. try
  280. strm.next_in := InBuf;
  281. strm.avail_in := InBytes;
  282. strm.next_out := OutBuf;
  283. strm.avail_out := OutBytes;
  284. DCheck(inflateInit_(strm, zlib_version, sizeof(strm)));
  285. try
  286. while DCheck(inflate(strm, Z_NO_FLUSH)) <> Z_STREAM_END do
  287. begin
  288. P := OutBuf;
  289. Inc(OutBytes, BufInc);
  290. ReallocMem(OutBuf, OutBytes);
  291. strm.next_out := Pointer(PtrUInt(OutBuf) + (PtrUInt(strm.next_out) - PtrUInt(P)));
  292. strm.avail_out := BufInc;
  293. end;
  294. finally
  295. DCheck(inflateEnd(strm));
  296. end;
  297. ReallocMem(OutBuf, strm.total_out);
  298. OutBytes := strm.total_out;
  299. except
  300. zlibFreeMem(nil, OutBuf);
  301. raise
  302. end;
  303. end;
  304. { TCustomZlibStream }
  305. constructor TCustomZLibStream.Create(Strm: TStream);
  306. begin
  307. inherited Create;
  308. FStrm := Strm;
  309. FStrmPos := Strm.Position;
  310. {$IFNDEF FPCPASZLIB}
  311. FZRec.zalloc := @zlibAllocMem;
  312. FZRec.zfree := @zlibFreeMem;
  313. {$ENDIF}
  314. end;
  315. procedure TCustomZLibStream.Progress(Sender: TObject);
  316. begin
  317. if Assigned(FOnProgress) then FOnProgress(Sender);
  318. end;
  319. { TCompressionStream }
  320. constructor TCompressionStream.Create(CompressionLevel: TCompressionLevel;
  321. Dest: TStream);
  322. const
  323. Levels: array [TCompressionLevel] of ShortInt =
  324. (Z_NO_COMPRESSION, Z_BEST_SPEED, Z_DEFAULT_COMPRESSION, Z_BEST_COMPRESSION);
  325. begin
  326. inherited Create(Dest);
  327. FZRec.next_out := @FBuffer;
  328. FZRec.avail_out := sizeof(FBuffer);
  329. CCheck(deflateInit_(FZRec, Levels[CompressionLevel], zlib_version, sizeof(FZRec)));
  330. end;
  331. destructor TCompressionStream.Destroy;
  332. begin
  333. FZRec.next_in := nil;
  334. FZRec.avail_in := 0;
  335. try
  336. if FStrm.Position <> FStrmPos then FStrm.Position := FStrmPos;
  337. while (CCheck(deflate(FZRec, Z_FINISH)) <> Z_STREAM_END)
  338. and (FZRec.avail_out = 0) do
  339. begin
  340. FStrm.WriteBuffer(FBuffer, sizeof(FBuffer));
  341. FZRec.next_out := @FBuffer;
  342. FZRec.avail_out := sizeof(FBuffer);
  343. end;
  344. if FZRec.avail_out < sizeof(FBuffer) then
  345. FStrm.WriteBuffer(FBuffer, sizeof(FBuffer) - FZRec.avail_out);
  346. finally
  347. deflateEnd(FZRec);
  348. end;
  349. inherited Destroy;
  350. end;
  351. function TCompressionStream.Read(var Buffer; Count: Longint): Longint;
  352. begin
  353. raise ECompressionError.Create('Invalid stream operation');
  354. end;
  355. function TCompressionStream.Write(const Buffer; Count: Longint): Longint;
  356. begin
  357. FZRec.next_in := @Buffer;
  358. FZRec.avail_in := Count;
  359. if FStrm.Position <> FStrmPos then FStrm.Position := FStrmPos;
  360. while (FZRec.avail_in > 0) do
  361. begin
  362. CCheck(deflate(FZRec, 0));
  363. if FZRec.avail_out = 0 then
  364. begin
  365. FStrm.WriteBuffer(FBuffer, sizeof(FBuffer));
  366. FZRec.next_out := @FBuffer;
  367. FZRec.avail_out := sizeof(FBuffer);
  368. FStrmPos := FStrm.Position;
  369. Progress(Self);
  370. end;
  371. end;
  372. Result := Count;
  373. end;
  374. function TCompressionStream.Seek(Offset: Longint; Origin: Word): Longint;
  375. begin
  376. if (Offset = 0) and (Origin = soFromCurrent) then
  377. Result := FZRec.total_in
  378. else
  379. raise ECompressionError.Create('Invalid stream operation');
  380. end;
  381. function TCompressionStream.GetCompressionRate: Single;
  382. begin
  383. if FZRec.total_in = 0 then
  384. Result := 0
  385. else
  386. Result := (1.0 - (FZRec.total_out / FZRec.total_in)) * 100.0;
  387. end;
  388. { TDecompressionStream }
  389. constructor TDecompressionStream.Create(Source: TStream);
  390. begin
  391. inherited Create(Source);
  392. FZRec.next_in := @FBuffer;
  393. FZRec.avail_in := 0;
  394. DCheck(inflateInit_(FZRec, zlib_version, sizeof(FZRec)));
  395. end;
  396. destructor TDecompressionStream.Destroy;
  397. begin
  398. inflateEnd(FZRec);
  399. inherited Destroy;
  400. end;
  401. function TDecompressionStream.Read(var Buffer; Count: Longint): Longint;
  402. begin
  403. FZRec.next_out := @Buffer;
  404. FZRec.avail_out := Count;
  405. if FStrm.Position <> FStrmPos then FStrm.Position := FStrmPos;
  406. while (FZRec.avail_out > 0) do
  407. begin
  408. if FZRec.avail_in = 0 then
  409. begin
  410. FZRec.avail_in := FStrm.Read(FBuffer, sizeof(FBuffer));
  411. if FZRec.avail_in = 0 then
  412. begin
  413. Result := Count - Integer(FZRec.avail_out);
  414. Exit;
  415. end;
  416. FZRec.next_in := @FBuffer;
  417. FStrmPos := FStrm.Position;
  418. Progress(Self);
  419. end;
  420. CCheck(inflate(FZRec, 0));
  421. end;
  422. Result := Count;
  423. end;
  424. function TDecompressionStream.Write(const Buffer; Count: Longint): Longint;
  425. begin
  426. raise EDecompressionError.Create('Invalid stream operation');
  427. end;
  428. function TDecompressionStream.Seek(Offset: Longint; Origin: Word): Longint;
  429. var
  430. I: Integer;
  431. Buf: array [0..4095] of Byte;
  432. begin
  433. if (Offset = 0) and (Origin = soFromBeginning) then
  434. begin
  435. DCheck(inflateReset(FZRec));
  436. FZRec.next_in := @FBuffer;
  437. FZRec.avail_in := 0;
  438. FStrm.Position := 0;
  439. FStrmPos := 0;
  440. end
  441. else if ( (Offset >= 0) and (Origin = soFromCurrent)) or
  442. ( ((Offset - Integer(FZRec.total_out)) > 0) and (Origin = soFromBeginning)) then
  443. begin
  444. if Origin = soFromBeginning then Dec(Offset, FZRec.total_out);
  445. if Offset > 0 then
  446. begin
  447. for I := 1 to Offset div sizeof(Buf) do
  448. ReadBuffer(Buf, sizeof(Buf));
  449. ReadBuffer(Buf, Offset mod sizeof(Buf));
  450. end;
  451. end
  452. else
  453. raise EDecompressionError.Create('Invalid stream operation');
  454. Result := FZRec.total_out;
  455. end;
  456. end.