storage = Storage::disk('local'); $this->path = $path; $this->type = $type; } /** * @return mixed * @throws \Exception */ public function execute() { if ($this->type === 'txt') { return $this->path; } if ($this->type === 'pdf') { // $this->convertPdfToText(); $this->convertPdfToMD(); // $this->getHtmlContentsFromPdfWithImages(); return $this->path; } if ($this->type !== 'docx') { $this->convertToDocx(); } $this->convertDocumentToText(); //$this->convertToHtml(); return $this->path; } /** * Convert doc,dot,rtf,odt,pdf,docx to docx * * * @return string|void */ private function convertToDocx() { (new Process(['export HOME=' . env('USER_HOME_PATH')]))->run(); /** * Convert doc,dot,rtf,odt to docx */ $process = new Process([ 'soffice', '--headless', '--convert-to', 'docx', $this->storage->path($this->path), '--outdir', $this->storage->path('contracts') ]); $process->run(); if (!$process->isSuccessful()) { throw new ProcessFailedException($process); } $this->storage->delete($this->path); $this->path = str_replace(".$this->type", '.docx', $this->path); } /** * Convert docx file to text * * @return void */ private function convertDocumentToText() { (new Process(['export HOME=' . env('USER_HOME_PATH')]))->run(); $process = new Process([ 'soffice', '--headless', '--convert-to', 'txt', $this->storage->path($this->path), '--outdir', $this->storage->path('contracts') ]); $process->run(); if (!$process->isSuccessful()) { throw new ProcessFailedException($process); } $this->storage->delete($this->path); $this->path = str_replace(['.docx', '.bin'], '.txt', $this->path); } protected function convertPdfToText() { $this->prepareForConvertPDF(); $images = $this->getImagesFromPDF(); $contents = $this->getTextContentsFromPDF(); if (!$contents && count($images) === 0) { throw new \Exception('Could not read from file.'); } // Handle images and image contents. if (count($images) > 0) { foreach ($images as $image) { try { $ocr = new OCR($this->storage->path($image)); $imageContents = $ocr->execute(); $contents = $contents . "\n" . $imageContents; } catch (\Exception $exception) { \Illuminate\Support\Facades\Log::info('something wrong: ' . $exception->getMessage()); } } $dir = str_replace('.pdf', '', $this->path); $this->storage->deleteDirectory($dir); } $this->storage->delete($this->path); $this->path = str_replace('.pdf', '.txt', $this->path); $this->storage->put($this->path, $contents); } protected function convertPdfToMD() { // $this->prepareForConvertPDF(); $result = $this->getContentsFromPdf(); if ( ! $result['has_images'] && ! $result['has_text']) { throw new \Exception('Cannot get pdf file contents.'); } if ($result['has_text']) { if ($result['has_images']) { // Both text and images. throw new \Exception('Not supported for now.'); } // Delete directory because the contents are in the '$result' variable. $this->storage->deleteDirectory($this->path); $mdContents = ''; foreach ($result['htmls'] as $html) { $converter = new HtmlConverter(); $converter->getConfig()->setOption('strip_tags', true); $contents = $converter->convert($html); $mdContents = $mdContents . $contents; } $this->path = "$this->path.md"; $this->storage->put($this->path, $mdContents); return; } // Only contains images. $imagesContent = ''; $files = $this->storage->allFiles($this->path); foreach ($files as $file) { // Only get the image files from the directory, it may contain some empty html files too. if (in_array(pathinfo($file, PATHINFO_EXTENSION), ['jpg', 'png'])) { $ocr = new OCR($this->storage->path($file)); $imagesContent = $imagesContent . $ocr->execute(); } } \Illuminate\Support\Facades\Log::info('============================'); \Illuminate\Support\Facades\Log::info($this->path); // We are done with the images processing, delete directory. $this->storage->deleteDirectory($this->path); $this->path = "$this->path.md"; \Illuminate\Support\Facades\Log::info($this->path); \Illuminate\Support\Facades\Log::info('++++++++++++++++++++++++++'); $this->storage->put($this->path, $imagesContent); } private function convertToHtml() { (new Process(['export HOME=' . env('USER_HOME_PATH')]))->run(); $process = new Process([ 'soffice', '--headless', '--convert-to', 'html:HTML:EmbedImages', $this->storage->path($this->path), '--outdir', $this->storage->path('contracts') ]); $process->run(); if (!$process->isSuccessful()) { throw new ProcessFailedException($process); } $this->storage->delete($this->path); $this->path = str_replace(".$this->type", '.html', $this->path); } private function convertToXML() { //Convert the file to xml using pdftohtml to xml and run a python scrypt to fix the paragraphs $process = new Process([ 'pdftohtml', '-xml', '-i', $this->storage->path($this->path) ]); $process->run(); if (!$process->isSuccessful()) { throw new ProcessFailedException($process); } $this->storage->delete($this->path); $this->path = str_replace(".$this->type", '.xml', $this->path); } protected function prepareForConvertPDF() { (new Process(['export HOME=' . env('USER_HOME_PATH')]))->run(); $process = new Process([ 'pip3', 'install', 'pdftotext', ]); $process->run(); if (!$process->isSuccessful()) { throw new ProcessFailedException($process); } } protected function getImagesFromPDF() { $dir = str_replace('.pdf', '', $this->path); $this->storage->makeDirectory($dir); $process = new Process([ 'pdfimages', '-p', $this->storage->path($this->path), '-tiff', $this->storage->path("$dir/ocr") ]); $process->run(); if (!$process->isSuccessful()) { throw new ProcessFailedException($process); } return $this->storage->allFiles($dir); } protected function getTextContentsFromPDF() { $outputPath = $this->storage->path(str_replace('.pdf', '.txt', $this->path)); $process = new Process([ 'python3', storage_path('scripts' . DIRECTORY_SEPARATOR . 'parse-pdf.py'), '-i', $this->storage->path($this->path), '-o', $outputPath ]); $process->run(); if (!$process->isSuccessful()) { throw new ProcessFailedException($process); } return file_get_contents($outputPath); } protected function getHtmlContentsFromPdfWithImages() { $dirName = str_replace('.pdf', '', $this->path); $this->storage->makeDirectory($dirName); $outputPath = $this->storage->path("$dirName/html"); $process = new Process([ 'pdftohtml', '-noframes', $this->storage->path($this->path), $outputPath ]); $process->run(); if (!$process->isSuccessful()) { throw new ProcessFailedException($process); } $this->storage->delete($this->path); $this->path = $dirName; $converter = new HtmlConverter(); $converter->getConfig()->setOption('strip_tags', true); $files = $this->storage->allFiles($this->path); $htmlFileIndex = null; foreach ($files as $index => $file) { // if (pathinfo($file, PATHINFO_BASENAME) === 'html-html.html') { // if (pathinfo($file, PATHINFO_EXTENSION) === 'html') { if (pathinfo($file, PATHINFO_BASENAME) === 'html.html') { $htmlFileIndex = $index; break; } } $htmlContents = $this->storage->get($files[$htmlFileIndex]); $contents = $converter->convert($htmlContents); // $this->storage->deleteDirectory($this->path); $this->path = "$this->path.md"; $this->storage->put($this->path, $contents); dd(3); } protected function getContentsFromPdf() { $dirName = str_replace('.pdf', '', $this->path); $this->storage->makeDirectory($dirName); $outputPath = $this->storage->path("$dirName/html"); $process = new Process([ 'pdftohtml', '-xml', $this->storage->path($this->path), $outputPath ]); $process->run(); if (!$process->isSuccessful()) { throw new ProcessFailedException($process); } $this->storage->delete($this->path); $this->path = $dirName; $contents = $this->storage->get("$this->path/html.xml"); $xml = simplexml_load_string($contents); $fonts = []; foreach ($xml->page as $page) { foreach ($page as $p) { if ($p->getName() === 'fontspec') { $fonts[(int) $p['id']]['family'] = (string) $p['family']; $fonts[(int) $p['id']]['size'] = (string) $p['size']; $fonts[(int) $p['id']]['color'] = (string) $p['color']; } } } $htmls = []; $hasImages = false; $hasText = false; try { foreach ($xml->page as $page) { $html = ''; $previousP = null; foreach ($page as $p) { if ($p->getName() == 'image') { $html = $html . ''; $hasImages = true; } if ($p->getName() == 'text') { $id = (int) $p['font']; $font_size = $fonts[$id]['size']; $font_color = $fonts[$id]['color']; $font_family = $fonts[$id]['family']; $style = ''; $style = $style . 'position: absolute;'; $style = $style . "color: $font_color;"; $style = $style . "font-family: $font_family;"; $style = $style . "font-weight: 900;"; $style = $style . "width: " . $p['width'] . "px;"; $style = $style . "height: " . $p['height'] . "px;"; $style = $style . "top: " . $p['top'] . "px;"; $style = $style . "left: " . $p['left'] . "px;"; // $style = $style . "font-size: $font_size" . "px;"; if ($p->i) { $content = '' . $p->i . ''; } else if ($p->b) { $content = '' . $p->b . ''; } else { $content = $p; } // @TODO Must chain paragraphs if top are almost same. $tag = $this->getTag($p, $previousP, $font_size); $html = $html . '<' . $tag . ' style="' . $style . '">' . $content . ''; $hasText = true; } $previousP = $p; } $htmls[] = '' . $html . ''; } } catch (\Exception $exception) { \Illuminate\Support\Facades\Log::info($exception->getTraceAsString()); } return [ 'has_images' => $hasImages, 'has_text' => $hasText, 'htmls' => $htmls, ]; } protected function getTag($p, $previousP, $size) { if ($size > 24) { return 'h1'; } if ($size > 18) { return 'h2'; } if ($size > 16) { return 'h3'; } if ($previousP && $p['top'] - $previousP['top'] <= 5) { return 'span'; } return 'p'; } }