diff --git a/app/Console/Commands/RunSearchDisplace.php b/app/Console/Commands/RunSearchDisplace.php index 8b0ca22..760cabd 100644 --- a/app/Console/Commands/RunSearchDisplace.php +++ b/app/Console/Commands/RunSearchDisplace.php @@ -48,9 +48,9 @@ class RunSearchDisplace extends Command $resultedDocumentPath = $pathDetails['dirname'] . '/' . $pathDetails['filename'] . '-displaced.md'; - $this->storeSearchers($id, $searchers, $resultedDocumentPath); - try { + $this->storeSearchers($id, $searchers, $resultedDocumentPath); + $sendToIngest = new SendDocument(); $sendToIngest->execute($id, [ @@ -89,14 +89,22 @@ class RunSearchDisplace extends Command protected function getSearchersFromList($searchers) { + $storage = Storage::disk('local'); + $list = []; foreach ($searchers as $searcher) { $result = explode(':', $searcher); + $searcherPath = 'searchers/' . $result[0] . '.json'; + + if ( ! $storage->exists($searcherPath)) { + throw new \Exception('Searcher does not exist'); + } + $list[] = [ - 'key' => $result[0], - 'replace_with' => $result[1], + 'content' => json_decode($storage->get($searcherPath), true), + 'replace_with' => count($result) > 1 ? $result[1] : '', ]; } @@ -119,7 +127,7 @@ class RunSearchDisplace extends Command $contents = file_get_contents($path); if ( ! $contents) { - throw new \Exception('Something went wrong when tried reading from file.'); + throw new \Exception('There is no data in the searcher JSON file.'); } return [ diff --git a/app/Http/Controllers/FileController.php b/app/Http/Controllers/FileController.php index 9421c0d..75c9964 100644 --- a/app/Http/Controllers/FileController.php +++ b/app/Http/Controllers/FileController.php @@ -18,6 +18,15 @@ class FileController extends Controller */ public function create(): JsonResponse { + request()->validate([ + 'file' => [ + 'required', + 'file', + 'max:10000', + 'mimes:doc,dot,docx,dotx,docm,dotm,odt,rtf,pdf,txt', + ], + ]); + try { /** @var UploadedFile $file */ $file = request()->file('file'); diff --git a/app/SearchDisplace/Ingest/HandleReceivedDocument.php b/app/SearchDisplace/Ingest/HandleReceivedDocument.php index 543bab8..ec2bb08 100644 --- a/app/SearchDisplace/Ingest/HandleReceivedDocument.php +++ b/app/SearchDisplace/Ingest/HandleReceivedDocument.php @@ -34,7 +34,7 @@ class HandleReceivedDocument IngestDocumentReceived::dispatch($this->id); } catch (\Exception $exception) { - \Illuminate\Support\Facades\Log::info('exception. :' . $exception->getMessage()); + \Illuminate\Support\Facades\Log::info('Exception: ' . $exception->getTraceAsString()); } } } diff --git a/app/SearchDisplace/Ingest/SendDocument.php b/app/SearchDisplace/Ingest/SendDocument.php index 72a7c00..d36a4eb 100644 --- a/app/SearchDisplace/Ingest/SendDocument.php +++ b/app/SearchDisplace/Ingest/SendDocument.php @@ -3,6 +3,7 @@ namespace App\SearchDisplace\Ingest; use GuzzleHttp\Client; +use GuzzleHttp\Exception\ClientException; class SendDocument { @@ -32,30 +33,46 @@ class SendDocument } } + /** + * Send request to Ingest. + * + * @param $id + * @param $document + * @return mixed + * @throws \GuzzleHttp\Exception\GuzzleException + */ public function sendRequest($id, $document) { $client = new Client(); - $response = $client->request('post', $this->url, [ - 'headers' => [ - 'Accept' => 'application/json', - ], + try { + $response = $client->request('post', $this->url, [ + 'headers' => [ + 'Accept' => 'application/json', + ], + + 'multipart' => [ + [ + 'name' => 'id', + 'contents' => $id, + ], - 'multipart' => [ - [ - 'name' => 'id', - 'contents' => $id, + [ + 'name' => 'document', + 'contents' => fopen($document['path'], 'r'), + 'filename' => $document['name'], + 'Content-type' => $document['type'], + ] ], + ]); - [ - 'name' => 'document', - 'contents' => fopen($document['path'], 'r'), - 'filename' => $document['name'], - 'Content-type' => $document['type'], - ] - ], - ]); + return json_decode($response->getBody()->getContents(), true); + } catch (ClientException $clientException) { + $error = json_decode($clientException->getResponse()->getBody()->getContents(), true); - return json_decode($response->getBody()->getContents(), true); + throw new \Exception($error['message']); + } catch (\Exception $exception) { + throw $exception; + } } } diff --git a/app/SearchDisplace/SearchAndDisplaceFromFiles.php b/app/SearchDisplace/SearchAndDisplaceFromFiles.php index cb52eeb..382f966 100644 --- a/app/SearchDisplace/SearchAndDisplaceFromFiles.php +++ b/app/SearchDisplace/SearchAndDisplaceFromFiles.php @@ -23,35 +23,35 @@ class SearchAndDisplaceFromFiles public function execute() { - if ( ! $this->storage->exists($this->documentFilePath) || - ! $this->storage->exists($this->infoFilePath) - ) { + if ( ! $this->storage->exists($this->documentFilePath) || ! $this->storage->exists($this->infoFilePath)) { // Handle this case, must report result to user. return; } - $documentContent = $this->storage->get($this->documentFilePath); - $searchersContent = json_decode($this->storage->get($this->infoFilePath), true); + try { + $documentContent = $this->storage->get($this->documentFilePath); + $searchersContent = json_decode($this->storage->get($this->infoFilePath), true); - $documentPath = $searchersContent['document_path']; - $searchers = $searchersContent['searchers']; + $documentPath = $searchersContent['document_path']; + $searchers = $searchersContent['searchers']; - $this->storage->put($this->infoFilePath, json_encode($searchers[0]['content'])); + $this->storage->put($this->infoFilePath, json_encode($searchers[0]['content'])); - $searchAndDisplace = new SearchAndDisplace($documentContent, [ - 'searchers' => [ - [ - 'key' => $this->id, - 'replace_with' => $searchers[0]['replace_with'], - ] - ], - ]); + $searchAndDisplace = new SearchAndDisplace($documentContent, [ + 'searchers' => [ + [ + 'key' => $this->id, + 'replace_with' => $searchers[0]['replace_with'], + ] + ], + ]); - try { $result = $searchAndDisplace->execute(); file_put_contents($documentPath, $result['content']); } catch (\Exception $exception) { + \Illuminate\Support\Facades\Log::info('EXCEPTION: ' . $exception->getMessage()); + return; } finally { $this->storage->delete($this->documentFilePath); diff --git a/app/SearchDisplace/Searchers/Searcher.php b/app/SearchDisplace/Searchers/Searcher.php index 2b56b80..9df5a49 100644 --- a/app/SearchDisplace/Searchers/Searcher.php +++ b/app/SearchDisplace/Searchers/Searcher.php @@ -99,6 +99,10 @@ class Searcher return $this->handleExpression($searcher['expression'], $content, $mustMatchStartAndEnd); } + if (array_key_exists('id', $searcher) && $this->ducklingMapper->has($searcher['id'])) { + return $this->applyDucklingSearcher($content, $searcher['id']); + } + throw new \Exception('Invalid searcher.'); } @@ -120,7 +124,7 @@ class Searcher ] ]; - foreach ($searchers as $row) { + foreach ($searchers as $index => $row) { $newSerialSearchersResults = []; foreach ($serialSearchersResults as $serialSearcherItem) { diff --git a/demo-cli/README.md b/demo-cli/README.md new file mode 100644 index 0000000..7de20a5 --- /dev/null +++ b/demo-cli/README.md @@ -0,0 +1,36 @@ +# Running S&D via CLI + +### Command +`php artisan sd:run {path} {searchers*}` + +The command accepts two arguments: +- path: The path to the document file on which the Search&Displace will run +- searchers: This argument can be one of the following two types: + - file searchers: the argument must only have one group in the format 'path:replace_with', where path is the path to a **valid JSON** file + - inline searchers: the argument can have multiple groups of inline searchers in the format 'key: replace_with'. + The 'key' represents a valid searcher found in the 'storage/app/searchers' directory, without the '.json' extension. + +The 'replace_with' value is optional, not using it will remove the found text strings. + +The resulted Markdown document will be created in the same directory as the input document file. + +### Examples +Note! These examples work when running the command from the root app directory, otherwise you have to +input the correct paths in the command, including for the 'artisan' file. + +- Using valid JSON file searcher and removing all strings found +`php artisan sd:run ./demo-cli/demo_document.pdf ./demo-cli/demo_searcher.json` + +- Using valid JSON file searcher and replacing all strings found with the string 'EMAIL' + `php artisan sd:run ./demo-cli/demo_document.pdf ./demo-cli/demo_searcher.json:EMAIL` + +- Using valid searcher key (which exists in the directory 'storage/app/searchers') and removing all strings found + `php artisan sd:run ./demo-cli/demo_document.pdf demo_searcher:EMAIL` + +- Using valid searcher key (which exists in the directory 'storage/app/searchers') and replacing all strings found with the string 'EMAIL' + `php artisan sd:run ./demo-cli/demo_document.pdf demo_searcher:EMAIL` + + + + + diff --git a/demo-cli/demo_document.pdf b/demo-cli/demo_document.pdf new file mode 100644 index 0000000..e6ac65e Binary files /dev/null and b/demo-cli/demo_document.pdf differ diff --git a/demo-cli/demo_searcher.json b/demo-cli/demo_searcher.json new file mode 100644 index 0000000..acf6d78 --- /dev/null +++ b/demo-cli/demo_searcher.json @@ -0,0 +1,31 @@ +{ + "id": "Demo Searcher", + "name": "Emails containing john", + "description": "Finds email addresses and then filters out those which don't contain the string 'john'.", + "rows": [ + [ + { + "name": "Email", + "id": "email", + "description": null, + "type": "duckling" + } + ], + + [ + { + "id": "e97aa284061b58311423d4b0cccf596a_Contains john", + "name": "Contains john", + "description": null, + "rows": [ + [ + { + "expression": ".*john.*" + } + ] + ], + "type": "custom" + } + ] + ] +} diff --git a/public/js/app.js b/public/js/app.js index 8808c13..a46b078 100644 --- a/public/js/app.js +++ b/public/js/app.js @@ -3152,6 +3152,35 @@ AppHeader = (0,tslib__WEBPACK_IMPORTED_MODULE_1__.__decorate)([vue_class_compone /***/ }), +/***/ "./resources/js/SearchDisplace/helpers.ts": +/*!************************************************!*\ + !*** ./resources/js/SearchDisplace/helpers.ts ***! + \************************************************/ +/***/ ((__unused_webpack_module, __webpack_exports__, __webpack_require__) => { + +"use strict"; +__webpack_require__.r(__webpack_exports__); +/* harmony export */ __webpack_require__.d(__webpack_exports__, { +/* harmony export */ "isServerError": () => (/* binding */ isServerError), +/* harmony export */ "getServerErrorMessage": () => (/* binding */ getServerErrorMessage) +/* harmony export */ }); +function _typeof(obj) { "@babel/helpers - typeof"; if (typeof Symbol === "function" && typeof Symbol.iterator === "symbol") { _typeof = function _typeof(obj) { return typeof obj; }; } else { _typeof = function _typeof(obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; }; } return _typeof(obj); } + +function isServerError(e) { + return e && _typeof(e) === 'object' && e.hasOwnProperty('response') && e.response && e.response.hasOwnProperty('data') && e.response.data; +} +function getServerErrorMessage(e) { + var error = e.response.data; + + if (error.hasOwnProperty('message')) { + return error.message; + } + + return ''; +} + +/***/ }), + /***/ "./resources/js/app.ts": /*!*****************************!*\ !*** ./resources/js/app.ts ***! @@ -3353,8 +3382,9 @@ __webpack_require__.r(__webpack_exports__); /* harmony export */ }); /* harmony import */ var _babel_runtime_regenerator__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! @babel/runtime/regenerator */ "./node_modules/@babel/runtime/regenerator/index.js"); /* harmony import */ var _babel_runtime_regenerator__WEBPACK_IMPORTED_MODULE_0___default = /*#__PURE__*/__webpack_require__.n(_babel_runtime_regenerator__WEBPACK_IMPORTED_MODULE_0__); -/* harmony import */ var tslib__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! tslib */ "./node_modules/tslib/tslib.es6.js"); +/* harmony import */ var tslib__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(/*! tslib */ "./node_modules/tslib/tslib.es6.js"); /* harmony import */ var vue_property_decorator__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! vue-property-decorator */ "./node_modules/vue-property-decorator/lib/vue-property-decorator.js"); +/* harmony import */ var _SearchDisplace_helpers__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! @/SearchDisplace/helpers */ "./resources/js/SearchDisplace/helpers.ts"); function _typeof(obj) { "@babel/helpers - typeof"; if (typeof Symbol === "function" && typeof Symbol.iterator === "symbol") { _typeof = function _typeof(obj) { return typeof obj; }; } else { _typeof = function _typeof(obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; }; } return _typeof(obj); } @@ -3386,6 +3416,7 @@ function _getPrototypeOf(o) { _getPrototypeOf = Object.setPrototypeOf ? Object.g + var Home = /*#__PURE__*/function (_Vue) { _inherits(Home, _Vue); @@ -3450,7 +3481,7 @@ var Home = /*#__PURE__*/function (_Vue) { key: "uploadNewFile", value: function () { var _uploadNewFile = _asyncToGenerator( /*#__PURE__*/_babel_runtime_regenerator__WEBPACK_IMPORTED_MODULE_0___default().mark(function _callee2(file) { - var response; + var response, errors; return _babel_runtime_regenerator__WEBPACK_IMPORTED_MODULE_0___default().wrap(function _callee2$(_context2) { while (1) { switch (_context2.prev = _context2.next) { @@ -3471,23 +3502,48 @@ var Home = /*#__PURE__*/function (_Vue) { response = _context2.sent; this.fileUploaded = true; this.uploadResult = response; - _context2.next = 17; + _context2.next = 25; break; case 11: _context2.prev = 11; _context2.t0 = _context2["catch"](3); - console.log('Error uploading file: ', _context2.t0); - this.$toast.add({ - severity: 'error', - summary: 'Error!', - detail: 'There was an error uploading your file. Please try again later', - life: 3000 - }); this.uploading = false; this.fileUploaded = false; - case 17: + if (!(0,_SearchDisplace_helpers__WEBPACK_IMPORTED_MODULE_2__.isServerError)(_context2.t0)) { + _context2.next = 24; + break; + } + + if (!_context2.t0.response.data.hasOwnProperty('errors')) { + _context2.next = 21; + break; + } + + errors = _context2.t0.response.data.errors; + + if (!errors.hasOwnProperty('file')) { + _context2.next = 21; + break; + } + + this.error = errors.file[0]; + return _context2.abrupt("return"); + + case 21: + if (!_context2.t0.response.data.hasOwnProperty('message')) { + _context2.next = 24; + break; + } + + this.error = _context2.t0.response.data.message; + return _context2.abrupt("return"); + + case 24: + this.error = 'There was an error uploading your file. Please try again later.'; + + case 25: case "end": return _context2.stop(); } @@ -3511,11 +3567,11 @@ var Home = /*#__PURE__*/function (_Vue) { return Home; }(vue_property_decorator__WEBPACK_IMPORTED_MODULE_1__.Vue); -(0,tslib__WEBPACK_IMPORTED_MODULE_2__.__decorate)([(0,vue_property_decorator__WEBPACK_IMPORTED_MODULE_1__.Prop)({ +(0,tslib__WEBPACK_IMPORTED_MODULE_3__.__decorate)([(0,vue_property_decorator__WEBPACK_IMPORTED_MODULE_1__.Prop)({ default: [] })], Home.prototype, "searchers", void 0); -Home = (0,tslib__WEBPACK_IMPORTED_MODULE_2__.__decorate)([vue_property_decorator__WEBPACK_IMPORTED_MODULE_1__.Component], Home); +Home = (0,tslib__WEBPACK_IMPORTED_MODULE_3__.__decorate)([vue_property_decorator__WEBPACK_IMPORTED_MODULE_1__.Component], Home); /* harmony default export */ const __WEBPACK_DEFAULT_EXPORT__ = (Home); /***/ }), diff --git a/resources/js/components/Home/Home.ts b/resources/js/components/Home/Home.ts index ac26146..4084b40 100644 --- a/resources/js/components/Home/Home.ts +++ b/resources/js/components/Home/Home.ts @@ -1,5 +1,6 @@ import {Vue, Component, Prop} from 'vue-property-decorator'; import FileUploadResponse from '@/interfaces/responses/FileUploadResponse'; +import {isServerError} from "@/SearchDisplace/helpers"; @Component export default class Home extends Vue { @@ -41,17 +42,29 @@ export default class Home extends Vue { this.fileUploaded = true; this.uploadResult = response; - } catch (err) { - console.log('Error uploading file: ', err); - this.$toast.add({ - severity: 'error', - summary: 'Error!', - detail: 'There was an error uploading your file. Please try again later', - life: 3000 - }); - + } catch (e) { this.uploading = false; this.fileUploaded = false; + + if (isServerError(e)) { + if (e.response.data.hasOwnProperty('errors')) { + const errors = e.response.data.errors; + + if (errors.hasOwnProperty('file')) { + this.error = errors.file[0]; + + return; + } + } + + if (e.response.data.hasOwnProperty('message')) { + this.error = e.response.data.message; + + return; + } + } + + this.error = 'There was an error uploading your file. Please try again later.'; } }