Repo for the search and displace ingest module that takes odf, docx and pdf and transforms it into .md to be used with search and displace operations
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

190 lines
4.4 KiB

3 years ago
3 years ago
3 years ago
  1. <?php
  2. namespace App\Ingest;
  3. use Illuminate\Support\Facades\Storage;
  4. use Symfony\Component\Process\Exception\ProcessFailedException;
  5. use Symfony\Component\Process\Process;
  6. class Convertor
  7. {
  8. /**
  9. * @var \Illuminate\Contracts\Filesystem\Filesystem
  10. */
  11. private $storage;
  12. private $path;
  13. protected $type;
  14. public function __construct($path, $type)
  15. {
  16. $this->storage = Storage::disk('local');
  17. $this->path = $path;
  18. $this->type = $type;
  19. }
  20. public function execute()
  21. {
  22. if ($this->type === 'pdf') {
  23. $this->convertPdfToText();
  24. return $this->path;
  25. }
  26. if ($this->type !== 'docx') {
  27. $this->convertToDocx();
  28. }
  29. $this->convertDocumentToText();
  30. //$this->convertToHtml();
  31. return $this->path;
  32. }
  33. /**
  34. * Convert doc,dot,rtf,odt,pdf,docx to docx
  35. *
  36. *
  37. * @return string|void
  38. */
  39. private function convertToDocx()
  40. {
  41. (new Process(['export HOME=' . env('USER_HOME_PATH')]))->run();
  42. /**
  43. * Convert doc,dot,rtf,odt to docx
  44. */
  45. $process = new Process([
  46. 'soffice',
  47. '--headless',
  48. '--convert-to',
  49. 'docx',
  50. $this->storage->path($this->path),
  51. '--outdir',
  52. $this->storage->path('contracts')
  53. ]);
  54. $process->run();
  55. if (!$process->isSuccessful()) {
  56. throw new ProcessFailedException($process);
  57. }
  58. $this->storage->delete($this->path);
  59. $this->path = str_replace($this->type, 'docx', $this->path);
  60. }
  61. /**
  62. * Convert docx file to text
  63. *
  64. *
  65. * @return string|void
  66. */
  67. private function convertDocumentToText()
  68. {
  69. (new Process(['export HOME=' . env('USER_HOME_PATH')]))->run();
  70. $process = new Process([
  71. 'soffice',
  72. '--headless',
  73. '--convert-to',
  74. 'txt',
  75. $this->storage->path($this->path),
  76. '--outdir',
  77. $this->storage->path('contracts')
  78. ]);
  79. $process->run();
  80. if (!$process->isSuccessful()) {
  81. throw new ProcessFailedException($process);
  82. }
  83. $this->storage->delete($this->path);
  84. $this->path = str_replace(['.docx', '.bin'], '.txt', $this->path);
  85. }
  86. private function convertPdfToText()
  87. {
  88. $process = new Process([
  89. 'pip',
  90. 'install',
  91. "pdftotext"
  92. ]);
  93. $process->run();
  94. if (!$process->isSuccessful()) {
  95. throw new ProcessFailedException($process);
  96. }
  97. /**
  98. * Convert pdf to text
  99. */
  100. $process = new Process([
  101. 'python3',
  102. storage_path('scripts' . DIRECTORY_SEPARATOR . 'parse-pdf.py'),
  103. '-i',
  104. $this->storage->path($this->path),
  105. '-o',
  106. $this->storage->path(str_replace('.pdf', '.txt', $this->path))
  107. ]);
  108. $process->run();
  109. if (!$process->isSuccessful()) {
  110. throw new ProcessFailedException($process);
  111. }
  112. $this->storage->delete($this->path);
  113. $this->path = str_replace('pdf', 'txt', $this->path);
  114. }
  115. private function convertToHtml()
  116. {
  117. (new Process(['export HOME=' . env('USER_HOME_PATH')]))->run();
  118. $process = new Process([
  119. 'soffice',
  120. '--headless',
  121. '--convert-to',
  122. 'html:HTML:EmbedImages',
  123. $this->storage->path($this->path),
  124. '--outdir',
  125. $this->storage->path('contracts')
  126. ]);
  127. $process->run();
  128. if (!$process->isSuccessful()) {
  129. throw new ProcessFailedException($process);
  130. }
  131. $this->storage->delete($this->path);
  132. $this->path = str_replace($this->type, 'html', $this->path);
  133. }
  134. private function convertToXML()
  135. {
  136. //Convert the file to xml using pdftohtml to xml and run a python scrypt to fix the paragraphs
  137. $process = new Process([
  138. 'pdftohtml',
  139. '-xml',
  140. '-i',
  141. $this->storage->path($this->path)
  142. ]);
  143. $process->run();
  144. if (!$process->isSuccessful()) {
  145. throw new ProcessFailedException($process);
  146. }
  147. $this->storage->delete($this->path);
  148. $this->path = str_replace($this->type, 'xml', $this->path);
  149. }
  150. }