Orzu Ionut
3 years ago
16 changed files with 1084 additions and 157 deletions
-
2.idea/vcs.xml
-
4app/Http/Controllers/SearchAndDisplaceController.php
-
95app/Http/Controllers/SearchAndDisplaceOriginalDocumentController.php
-
70app/SearchDisplace/Ingest/HandleReceivedDocument.php
-
73app/SearchDisplace/Ingest/SendDataToRecreateDocument.php
-
16app/SearchDisplace/Ingest/SendDocument.php
-
3app/SearchDisplace/SearchAndDisplace.php
-
207app/SearchDisplace/SearchAndDisplaceOriginalDocument.php
-
509public/js/app.js
-
4resources/js/components/Home/Home.ts
-
1resources/js/components/Home/Home.vue
-
123resources/js/components/ProcessFile/ProcessFile.ts
-
9resources/js/components/ProcessFile/ProcessFile.vue
-
41resources/js/services/ApiService.ts
-
14routes/api.php
-
4routes/web.php
@ -1,6 +1,6 @@ |
|||
<?xml version="1.0" encoding="UTF-8"?> |
|||
<project version="4"> |
|||
<component name="VcsDirectoryMappings"> |
|||
<mapping directory="$PROJECT_DIR$" vcs="Git" /> |
|||
<mapping directory="" vcs="Git" /> |
|||
</component> |
|||
</project> |
@ -0,0 +1,95 @@ |
|||
<?php |
|||
|
|||
namespace App\Http\Controllers; |
|||
|
|||
use App\SearchDisplace\SearchAndDisplaceOriginalDocument; |
|||
use GuzzleHttp\Exception\BadResponseException; |
|||
|
|||
class SearchAndDisplaceOriginalDocumentController extends Controller |
|||
{ |
|||
public function store() |
|||
{ |
|||
request()->validate([ |
|||
'document' => [ |
|||
'required', |
|||
'file', |
|||
'max:10000', |
|||
'mimes:doc,dot,docx,dotx,docm,dotm,odt,rtf,pdf,txt', |
|||
], |
|||
|
|||
// Searchers is encoded.
|
|||
// 'searchers' => 'required|array',
|
|||
// 'searchers.*.key' => 'required',
|
|||
// 'searchers.*.type' => 'required|in:replace,displace',
|
|||
// 'searchers.*.value' => 'nullable',
|
|||
]); |
|||
|
|||
// Send document to Ingest to be processed as docx in order to get original data.
|
|||
|
|||
// After we get the response from Ingest apply S&D on the result.
|
|||
// After that send back to Ingest to recreate the original document.
|
|||
// After that Ingest will send webhook that it is done.
|
|||
// Download file from Ingest.
|
|||
// The interface will keep asking if the file is ready.
|
|||
// Once is ready return the response with the file.
|
|||
|
|||
try { |
|||
$handler = new SearchAndDisplaceOriginalDocument(); |
|||
|
|||
$id = $handler->start(request()->file('document'), json_decode(request()->get('searchers'), true)); |
|||
|
|||
return response()->json([ |
|||
'status' => 'ok', |
|||
'id' => $id, |
|||
]); |
|||
} catch (BadResponseException $e) { |
|||
return response()->json([ |
|||
'message' => $e->getMessage(), |
|||
'response' => $e->getResponse() |
|||
], 400); |
|||
} catch (\Exception $e) { |
|||
return response()->json([ |
|||
'message' => $e->getMessage(), |
|||
], 400); |
|||
} |
|||
} |
|||
|
|||
public function show($id) |
|||
{ |
|||
try { |
|||
$handler = new SearchAndDisplaceOriginalDocument(); |
|||
|
|||
if ($handler->hasFailed($id)) { |
|||
return response()->json([ |
|||
'status' => 'fail', |
|||
], 200); |
|||
} |
|||
|
|||
if ($handler->isInProgress($id)) { |
|||
return response()->json([ |
|||
'status' => 'processing', |
|||
], 200); |
|||
} |
|||
|
|||
return response()->json([ |
|||
'status' => 'success', |
|||
], 200); |
|||
} catch (\Exception $exception) { |
|||
return response()->json([ |
|||
'message' => $exception->getMessage(), |
|||
], 400); |
|||
} |
|||
} |
|||
|
|||
public function download($id) |
|||
{ |
|||
try { |
|||
$handler = new SearchAndDisplaceOriginalDocument(); |
|||
|
|||
return response()->download($handler->getDownloadPath($id)) |
|||
->deleteFileAfterSend(true); |
|||
} catch (\Exception $exception) { |
|||
abort(404); |
|||
} |
|||
} |
|||
} |
@ -0,0 +1,73 @@ |
|||
<?php |
|||
|
|||
namespace App\SearchDisplace\Ingest; |
|||
|
|||
use GuzzleHttp\Client; |
|||
use GuzzleHttp\Exception\ClientException; |
|||
|
|||
class SendDataToRecreateDocument |
|||
{ |
|||
protected $url; |
|||
|
|||
public function __construct() |
|||
{ |
|||
$this->url = env('SD_INGEST_URL') . '/recreate-document'; |
|||
} |
|||
|
|||
/** |
|||
* |
|||
* @param $id |
|||
* @param $data |
|||
* @throws \GuzzleHttp\Exception\GuzzleException |
|||
* @throws \Exception |
|||
*/ |
|||
public function execute($id, $data) |
|||
{ |
|||
$response = $this->sendRequest($id, $data); |
|||
|
|||
if ( ! array_key_exists('status', $response)) { |
|||
throw new \Exception('Something went wrong.'); |
|||
} |
|||
|
|||
if ($response['status'] === 'fail') { |
|||
$message = array_key_exists('message', $response) |
|||
? $response['message'] |
|||
: 'Something went wrong.'; |
|||
|
|||
throw new \Exception($message); |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* Send request to Ingest to recreate document. |
|||
* |
|||
* @param $id |
|||
* @param $data |
|||
* @return mixed |
|||
* @throws \GuzzleHttp\Exception\GuzzleException |
|||
* @throws \Exception |
|||
*/ |
|||
public function sendRequest($id, $data) |
|||
{ |
|||
$client = new Client(); |
|||
|
|||
try { |
|||
$response = $client->request('post', $this->url, [ |
|||
'headers' => [ |
|||
'Accept' => 'application/json', |
|||
], |
|||
|
|||
'form_params' => [ |
|||
'id' => $id, |
|||
'data' => json_encode($data), |
|||
], |
|||
]); |
|||
|
|||
return json_decode($response->getBody()->getContents(), true); |
|||
} catch (ClientException $clientException) { |
|||
$error = json_decode($clientException->getResponse()->getBody()->getContents(), true); |
|||
|
|||
throw new \Exception($error['message']); |
|||
} |
|||
} |
|||
} |
@ -0,0 +1,207 @@ |
|||
<?php |
|||
|
|||
namespace App\SearchDisplace; |
|||
|
|||
use App\SearchDisplace\Ingest\SendDataToRecreateDocument; |
|||
use App\SearchDisplace\Ingest\SendDocument; |
|||
use Illuminate\Support\Facades\Storage; |
|||
|
|||
class SearchAndDisplaceOriginalDocument |
|||
{ |
|||
/** |
|||
* |
|||
* @throws \Exception |
|||
*/ |
|||
public function start($document, $searchers) |
|||
{ |
|||
$id = time() . '_' . pathinfo($document->getClientOriginalName(), PATHINFO_FILENAME); |
|||
|
|||
$this->storeSearchers($id, $searchers); |
|||
$this->sendDocumentToIngest($id, $document); |
|||
|
|||
return $id; |
|||
} |
|||
|
|||
/** |
|||
* @param $id |
|||
* @param $contents |
|||
* @throws \GuzzleHttp\Exception\GuzzleException |
|||
*/ |
|||
public function applySD($id, $contents) |
|||
{ |
|||
$data = json_decode($contents['document'], true); |
|||
|
|||
try { |
|||
$searchAndDisplace = new SearchAndDisplace( |
|||
$data['contents']['text'], |
|||
|
|||
[ |
|||
'searchers' => $this->getSearchers($id), |
|||
] |
|||
); |
|||
|
|||
$result = $searchAndDisplace->execute(); |
|||
|
|||
// Update text.
|
|||
$x = $this->applyResultsOnIngestData($data['contents'], $result); |
|||
$data['contents'] = $x; |
|||
|
|||
$this->sendDataToIngestToRebuild($id, $data); |
|||
} catch (\Exception $exception) { |
|||
\Illuminate\Support\Facades\Log::info($exception->getMessage()); |
|||
} |
|||
} |
|||
|
|||
public function onIngestFail($id) |
|||
{ |
|||
$storage = Storage::disk('local'); |
|||
$directory = "contracts/$id"; |
|||
|
|||
$storage->deleteDirectory($directory); |
|||
} |
|||
|
|||
public function hasFailed($id) |
|||
{ |
|||
$storage = Storage::disk('local'); |
|||
$basePath = "contracts/$id"; |
|||
$filePath = $basePath . '-document.docx'; |
|||
|
|||
return ! $storage->exists($basePath) && ! $storage->exists($filePath); |
|||
} |
|||
|
|||
public function isInProgress($id) |
|||
{ |
|||
$storage = Storage::disk('local'); |
|||
$basePath = "contracts/$id"; |
|||
|
|||
// @TODO Set document extension.
|
|||
return $storage->exists($basePath) && ! $storage->exists($basePath . '-document.docx'); |
|||
} |
|||
|
|||
/** |
|||
* @param $id |
|||
* @return string |
|||
* @throws \Exception |
|||
*/ |
|||
public function getDownloadPath($id) |
|||
{ |
|||
$storage = Storage::disk('local'); |
|||
|
|||
if ($this->hasFailed($id) || $this->isInProgress($id)) { |
|||
throw new \Exception('Document is not processed.'); |
|||
} |
|||
|
|||
// @TODO Set document extension.
|
|||
return $storage->path('contracts/' . $id . '-document.docx'); |
|||
} |
|||
|
|||
protected function applyResultsOnIngestData($ingestData, $sdResult) |
|||
{ |
|||
$ingestData['text'] = $sdResult['content']; |
|||
|
|||
// Update index ranges.
|
|||
$indexes = []; |
|||
|
|||
// Use original start for key in order to have the indexes sorted ASC.
|
|||
foreach ($sdResult['indexes'] as $searcher => $searcherIndexes) { |
|||
foreach ($searcherIndexes as $index) { |
|||
$indexes[$index['original_start']] = $index; |
|||
} |
|||
} |
|||
|
|||
$lastOffset = 0; |
|||
|
|||
foreach ($ingestData['elements'] as $elementIndex => $element) { |
|||
$currentOffset = 0; |
|||
|
|||
foreach ($indexes as $i => $index) { |
|||
if ($index['original_start'] > $element['range_end']) { |
|||
break; |
|||
} |
|||
|
|||
if ($index['original_end'] < $element['range_start']) { |
|||
continue; |
|||
} |
|||
|
|||
if ( |
|||
$index['original_start'] >= $element['range_start'] && |
|||
$index['original_end'] <= $element['range_end'] |
|||
) { |
|||
$endDifference = ($index['end'] - $index['original_end']) - |
|||
($index['start'] - $index['original_start']); |
|||
|
|||
$ingestData['elements'][$elementIndex]['range_end'] += $endDifference; |
|||
$currentOffset += $endDifference; |
|||
|
|||
unset($indexes[$i]); |
|||
} |
|||
} |
|||
|
|||
$ingestData['elements'][$elementIndex]['range_start'] += $lastOffset; |
|||
$ingestData['elements'][$elementIndex]['range_end'] += $lastOffset; |
|||
|
|||
$lastOffset += $currentOffset; |
|||
} |
|||
|
|||
return $ingestData; |
|||
} |
|||
|
|||
protected function storeSearchers($id, $searchers) |
|||
{ |
|||
$storage = Storage::disk('local'); |
|||
$directory = "contracts/$id"; |
|||
$storage->makeDirectory($directory); |
|||
|
|||
$storage->put("$directory/searchers.json", json_encode($searchers)); |
|||
} |
|||
|
|||
/** |
|||
* |
|||
* @param $id |
|||
* @return string |
|||
* @throws \Illuminate\Contracts\Filesystem\FileNotFoundException |
|||
*/ |
|||
protected function getSearchers($id) |
|||
{ |
|||
$storage = Storage::disk('local'); |
|||
$directory = "contracts/$id"; |
|||
|
|||
$searchers = $storage->get("$directory/searchers.json"); |
|||
|
|||
if ( ! $searchers) { |
|||
throw new \Exception('Searchers do not exist.'); |
|||
} |
|||
|
|||
return json_decode($searchers, true); |
|||
} |
|||
|
|||
/** |
|||
* |
|||
* @param $id |
|||
* @param $document |
|||
* @throws \Exception |
|||
*/ |
|||
protected function sendDocumentToIngest($id, $document) |
|||
{ |
|||
$sendDocument = new SendDocument(); |
|||
|
|||
$sendDocument->execute($id, [ |
|||
'path' => $document->getRealPath(), |
|||
'type' => $document->getMimeType(), |
|||
'name' => $document->getClientOriginalName() |
|||
], 'original'); |
|||
} |
|||
|
|||
/** |
|||
* |
|||
* @param $id |
|||
* @param $data |
|||
* @throws \GuzzleHttp\Exception\GuzzleException |
|||
*/ |
|||
protected function sendDataToIngestToRebuild($id, $data) |
|||
{ |
|||
$handler = new SendDataToRecreateDocument(); |
|||
|
|||
$handler->execute($id, $data); |
|||
} |
|||
} |
Write
Preview
Loading…
Cancel
Save
Reference in new issue