Add demo for CLI. Various fixes and improvements.

app/Console/Commands/RunSearchDisplace.php

@@ -48,9 +48,9 @@ class RunSearchDisplace extends Command
 
         $resultedDocumentPath = $pathDetails['dirname'] . '/' . $pathDetails['filename'] . '-displaced.md';
 
-        $this->storeSearchers($id, $searchers, $resultedDocumentPath);
-
         try {
+            $this->storeSearchers($id, $searchers, $resultedDocumentPath);
+
             $sendToIngest = new SendDocument();
 
             $sendToIngest->execute($id, [
@@ -89,14 +89,22 @@ class RunSearchDisplace extends Command
 
     protected function getSearchersFromList($searchers)
     {
+        $storage = Storage::disk('local');
+
         $list = [];
 
         foreach ($searchers as $searcher) {
             $result = explode(':', $searcher);
 
+            $searcherPath = 'searchers/' . $result[0] . '.json';
+
+            if ( ! $storage->exists($searcherPath)) {
+                throw new \Exception('Searcher does not exist');
+            }
+
             $list[] = [
-                'key' => $result[0],
-                'replace_with' => $result[1],
+                'content' => json_decode($storage->get($searcherPath), true),
+                'replace_with' => count($result) > 1 ? $result[1] : '',
             ];
         }
 
@@ -119,7 +127,7 @@ class RunSearchDisplace extends Command
         $contents = file_get_contents($path);
 
         if ( ! $contents) {
-            throw new \Exception('Something went wrong when tried reading from file.');
+            throw new \Exception('There is no data in the searcher JSON file.');
         }
 
         return [

app/Http/Controllers/FileController.php

@@ -18,6 +18,15 @@ class FileController extends Controller
      */
     public function create(): JsonResponse
     {
+        request()->validate([
+            'file' => [
+                'required',
+                'file',
+                'max:10000',
+                'mimes:doc,dot,docx,dotx,docm,dotm,odt,rtf,pdf,txt',
+            ],
+        ]);
+
         try {
             /** @var UploadedFile $file */
             $file = request()->file('file');

app/SearchDisplace/Ingest/HandleReceivedDocument.php

@@ -34,7 +34,7 @@ class HandleReceivedDocument
 
             IngestDocumentReceived::dispatch($this->id);
         } catch (\Exception $exception) {
-            \Illuminate\Support\Facades\Log::info('exception. :' . $exception->getMessage());
+            \Illuminate\Support\Facades\Log::info('Exception: ' . $exception->getTraceAsString());
         }
     }
 }

app/SearchDisplace/Ingest/SendDocument.php

@@ -3,6 +3,7 @@
 namespace App\SearchDisplace\Ingest;
 
 use GuzzleHttp\Client;
+use GuzzleHttp\Exception\ClientException;
 
 class SendDocument
 {
@@ -32,30 +33,46 @@ class SendDocument
         }
     }
 
+    /**
+     * Send request to Ingest.
+     *
+     * @param $id
+     * @param $document
+     * @return mixed
+     * @throws \GuzzleHttp\Exception\GuzzleException
+     */
     public function sendRequest($id, $document)
     {
         $client = new Client();
 
-        $response = $client->request('post', $this->url, [
-            'headers' => [
-                'Accept' => 'application/json',
-            ],
+        try {
+            $response = $client->request('post', $this->url, [
+                'headers' => [
+                    'Accept' => 'application/json',
+                ],
+
+                'multipart' => [
+                    [
+                        'name' => 'id',
+                        'contents' => $id,
+                    ],
 
-            'multipart' => [
-                [
-                    'name' => 'id',
-                    'contents' => $id,
+                    [
+                        'name' => 'document',
+                        'contents' => fopen($document['path'], 'r'),
+                        'filename' => $document['name'],
+                        'Content-type' => $document['type'],
+                    ]
                 ],
+            ]);
 
-                [
-                    'name' => 'document',
-                    'contents' => fopen($document['path'], 'r'),
-                    'filename' => $document['name'],
-                    'Content-type' => $document['type'],
-                ]
-            ],
-        ]);
+            return json_decode($response->getBody()->getContents(), true);
+        } catch (ClientException $clientException) {
+            $error = json_decode($clientException->getResponse()->getBody()->getContents(), true);
 
-        return json_decode($response->getBody()->getContents(), true);
+            throw new \Exception($error['message']);
+        } catch (\Exception $exception) {
+            throw $exception;
+        }
     }
 }

app/SearchDisplace/SearchAndDisplaceFromFiles.php

@@ -23,35 +23,35 @@ class SearchAndDisplaceFromFiles
 
     public function execute()
     {
-        if ( ! $this->storage->exists($this->documentFilePath) ||
-            ! $this->storage->exists($this->infoFilePath)
-        ) {
+        if ( ! $this->storage->exists($this->documentFilePath) || ! $this->storage->exists($this->infoFilePath)) {
             // Handle this case, must report result to user.
             return;
         }
 
-        $documentContent = $this->storage->get($this->documentFilePath);
-        $searchersContent = json_decode($this->storage->get($this->infoFilePath), true);
+        try {
+            $documentContent = $this->storage->get($this->documentFilePath);
+            $searchersContent = json_decode($this->storage->get($this->infoFilePath), true);
 
-        $documentPath = $searchersContent['document_path'];
-        $searchers = $searchersContent['searchers'];
+            $documentPath = $searchersContent['document_path'];
+            $searchers = $searchersContent['searchers'];
 
-        $this->storage->put($this->infoFilePath, json_encode($searchers[0]['content']));
+            $this->storage->put($this->infoFilePath, json_encode($searchers[0]['content']));
 
-        $searchAndDisplace = new SearchAndDisplace($documentContent, [
-            'searchers' => [
-                [
-                    'key' => $this->id,
-                    'replace_with' => $searchers[0]['replace_with'],
-                ]
-            ],
-        ]);
+            $searchAndDisplace = new SearchAndDisplace($documentContent, [
+                'searchers' => [
+                    [
+                        'key' => $this->id,
+                        'replace_with' => $searchers[0]['replace_with'],
+                    ]
+                ],
+            ]);
 
-        try {
             $result = $searchAndDisplace->execute();
 
             file_put_contents($documentPath, $result['content']);
         } catch (\Exception $exception) {
+            \Illuminate\Support\Facades\Log::info('EXCEPTION: ' . $exception->getMessage());
+
             return;
         } finally {
             $this->storage->delete($this->documentFilePath);

app/SearchDisplace/Searchers/Searcher.php

@@ -99,6 +99,10 @@ class Searcher
             return $this->handleExpression($searcher['expression'], $content, $mustMatchStartAndEnd);
         }
 
+        if (array_key_exists('id', $searcher) && $this->ducklingMapper->has($searcher['id']))  {
+            return $this->applyDucklingSearcher($content, $searcher['id']);
+        }
+
         throw new \Exception('Invalid searcher.');
     }
 
@@ -120,7 +124,7 @@ class Searcher
             ]
         ];
 
-        foreach ($searchers as $row) {
+        foreach ($searchers as $index => $row) {
             $newSerialSearchersResults = [];
 
             foreach ($serialSearchersResults as $serialSearcherItem) {

demo-cli/README.md

@@ -0,0 +1,36 @@
+# Running S&D via CLI
+
+### Command
+`php artisan sd:run {path} {searchers*}`
+
+The command accepts two arguments:
+- path: The path to the document file on which the Search&Displace will run
+- searchers: This argument can be one of the following two types:
+    - file searchers: the argument must only have one group in the format 'path:replace_with', where path is the path to a **valid JSON** file
+    - inline searchers: the argument can have multiple groups of inline searchers in the format 'key: replace_with'. 
+      The 'key' represents a valid searcher found in the 'storage/app/searchers' directory, without the '.json' extension.
+
+The 'replace_with' value is optional, not using it will remove the found text strings.
+
+The resulted Markdown document will be created in the same directory as the input document file.
+
+### Examples
+Note! These examples work when running the command from the root app directory, otherwise you have to 
+input the correct paths in the command, including for the 'artisan' file.
+
+- Using valid JSON file searcher and removing all strings found
+`php artisan sd:run ./demo-cli/demo_document.pdf ./demo-cli/demo_searcher.json`
+
+- Using valid JSON file searcher and replacing all strings found with the string 'EMAIL'
+  `php artisan sd:run ./demo-cli/demo_document.pdf ./demo-cli/demo_searcher.json:EMAIL`
+
+- Using valid searcher key (which exists in the directory 'storage/app/searchers') and removing all strings found
+  `php artisan sd:run ./demo-cli/demo_document.pdf demo_searcher:EMAIL`
+
+- Using valid searcher key (which exists in the directory 'storage/app/searchers') and replacing all strings found with the string 'EMAIL'
+  `php artisan sd:run ./demo-cli/demo_document.pdf demo_searcher:EMAIL`
+
+
+
+
+

demo-cli/demo_searcher.json

@@ -0,0 +1,31 @@
+{
+    "id": "Demo Searcher",
+    "name": "Emails containing john",
+    "description": "Finds email addresses and then filters out those which don't contain the string 'john'.",
+    "rows": [
+        [
+            {
+                "name": "Email",
+                "id": "email",
+                "description": null,
+                "type": "duckling"
+            }
+        ],
+
+        [
+            {
+                "id": "e97aa284061b58311423d4b0cccf596a_Contains john",
+                "name": "Contains john",
+                "description": null,
+                "rows": [
+                    [
+                        {
+                            "expression": ".*john.*"
+                        }
+                    ]
+                ],
+                "type": "custom"
+            }
+        ]
+    ]
+}

public/js/app.js

@@ -3152,6 +3152,35 @@ AppHeader = (0,tslib__WEBPACK_IMPORTED_MODULE_1__.__decorate)([vue_class_compone
 
 /***/ }),
 
+/***/ "./resources/js/SearchDisplace/helpers.ts":
+/*!************************************************!*\
+  !*** ./resources/js/SearchDisplace/helpers.ts ***!
+  \************************************************/
+/***/ ((__unused_webpack_module, __webpack_exports__, __webpack_require__) => {
+
+"use strict";
+__webpack_require__.r(__webpack_exports__);
+/* harmony export */ __webpack_require__.d(__webpack_exports__, {
+/* harmony export */   "isServerError": () => (/* binding */ isServerError),
+/* harmony export */   "getServerErrorMessage": () => (/* binding */ getServerErrorMessage)
+/* harmony export */ });
+function _typeof(obj) { "@babel/helpers - typeof"; if (typeof Symbol === "function" && typeof Symbol.iterator === "symbol") { _typeof = function _typeof(obj) { return typeof obj; }; } else { _typeof = function _typeof(obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; }; } return _typeof(obj); }
+
+function isServerError(e) {
+  return e && _typeof(e) === 'object' && e.hasOwnProperty('response') && e.response && e.response.hasOwnProperty('data') && e.response.data;
+}
+function getServerErrorMessage(e) {
+  var error = e.response.data;
+
+  if (error.hasOwnProperty('message')) {
+    return error.message;
+  }
+
+  return '';
+}
+
+/***/ }),
+
 /***/ "./resources/js/app.ts":
 /*!*****************************!*\
   !*** ./resources/js/app.ts ***!
@@ -3353,8 +3382,9 @@ __webpack_require__.r(__webpack_exports__);
 /* harmony export */ });
 /* harmony import */ var _babel_runtime_regenerator__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! @babel/runtime/regenerator */ "./node_modules/@babel/runtime/regenerator/index.js");
 /* harmony import */ var _babel_runtime_regenerator__WEBPACK_IMPORTED_MODULE_0___default = /*#__PURE__*/__webpack_require__.n(_babel_runtime_regenerator__WEBPACK_IMPORTED_MODULE_0__);
-/* harmony import */ var tslib__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! tslib */ "./node_modules/tslib/tslib.es6.js");
+/* harmony import */ var tslib__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(/*! tslib */ "./node_modules/tslib/tslib.es6.js");
 /* harmony import */ var vue_property_decorator__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! vue-property-decorator */ "./node_modules/vue-property-decorator/lib/vue-property-decorator.js");
+/* harmony import */ var _SearchDisplace_helpers__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! @/SearchDisplace/helpers */ "./resources/js/SearchDisplace/helpers.ts");
 function _typeof(obj) { "@babel/helpers - typeof"; if (typeof Symbol === "function" && typeof Symbol.iterator === "symbol") { _typeof = function _typeof(obj) { return typeof obj; }; } else { _typeof = function _typeof(obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; }; } return _typeof(obj); }
 
 
@@ -3386,6 +3416,7 @@ function _getPrototypeOf(o) { _getPrototypeOf = Object.setPrototypeOf ? Object.g
 
 
 
+
 var Home = /*#__PURE__*/function (_Vue) {
   _inherits(Home, _Vue);
 
@@ -3450,7 +3481,7 @@ var Home = /*#__PURE__*/function (_Vue) {
     key: "uploadNewFile",
     value: function () {
       var _uploadNewFile = _asyncToGenerator( /*#__PURE__*/_babel_runtime_regenerator__WEBPACK_IMPORTED_MODULE_0___default().mark(function _callee2(file) {
-        var response;
+        var response, errors;
         return _babel_runtime_regenerator__WEBPACK_IMPORTED_MODULE_0___default().wrap(function _callee2$(_context2) {
           while (1) {
             switch (_context2.prev = _context2.next) {
@@ -3471,23 +3502,48 @@ var Home = /*#__PURE__*/function (_Vue) {
                 response = _context2.sent;
                 this.fileUploaded = true;
                 this.uploadResult = response;
-                _context2.next = 17;
+                _context2.next = 25;
                 break;
 
               case 11:
                 _context2.prev = 11;
                 _context2.t0 = _context2["catch"](3);
-                console.log('Error uploading file: ', _context2.t0);
-                this.$toast.add({
-                  severity: 'error',
-                  summary: 'Error!',
-                  detail: 'There was an error uploading your file. Please try again later',
-                  life: 3000
-                });
                 this.uploading = false;
                 this.fileUploaded = false;
 
-              case 17:
+                if (!(0,_SearchDisplace_helpers__WEBPACK_IMPORTED_MODULE_2__.isServerError)(_context2.t0)) {
+                  _context2.next = 24;
+                  break;
+                }
+
+                if (!_context2.t0.response.data.hasOwnProperty('errors')) {
+                  _context2.next = 21;
+                  break;
+                }
+
+                errors = _context2.t0.response.data.errors;
+
+                if (!errors.hasOwnProperty('file')) {
+                  _context2.next = 21;
+                  break;
+                }
+
+                this.error = errors.file[0];
+                return _context2.abrupt("return");
+
+              case 21:
+                if (!_context2.t0.response.data.hasOwnProperty('message')) {
+                  _context2.next = 24;
+                  break;
+                }
+
+                this.error = _context2.t0.response.data.message;
+                return _context2.abrupt("return");
+
+              case 24:
+                this.error = 'There was an error uploading your file. Please try again later.';
+
+              case 25:
               case "end":
                 return _context2.stop();
             }
@@ -3511,11 +3567,11 @@ var Home = /*#__PURE__*/function (_Vue) {
   return Home;
 }(vue_property_decorator__WEBPACK_IMPORTED_MODULE_1__.Vue);
 
-(0,tslib__WEBPACK_IMPORTED_MODULE_2__.__decorate)([(0,vue_property_decorator__WEBPACK_IMPORTED_MODULE_1__.Prop)({
+(0,tslib__WEBPACK_IMPORTED_MODULE_3__.__decorate)([(0,vue_property_decorator__WEBPACK_IMPORTED_MODULE_1__.Prop)({
   default: []
 })], Home.prototype, "searchers", void 0);
 
-Home = (0,tslib__WEBPACK_IMPORTED_MODULE_2__.__decorate)([vue_property_decorator__WEBPACK_IMPORTED_MODULE_1__.Component], Home);
+Home = (0,tslib__WEBPACK_IMPORTED_MODULE_3__.__decorate)([vue_property_decorator__WEBPACK_IMPORTED_MODULE_1__.Component], Home);
 /* harmony default export */ const __WEBPACK_DEFAULT_EXPORT__ = (Home);
 
 /***/ }),

resources/js/components/Home/Home.ts

@@ -1,5 +1,6 @@
 import {Vue, Component, Prop} from 'vue-property-decorator';
 import FileUploadResponse from '@/interfaces/responses/FileUploadResponse';
+import {isServerError} from "@/SearchDisplace/helpers";
 
 @Component
 export default class Home extends Vue {
@@ -41,17 +42,29 @@ export default class Home extends Vue {
 
             this.fileUploaded = true;
             this.uploadResult = response;
-        } catch (err) {
-            console.log('Error uploading file: ', err);
-            this.$toast.add({
-                severity: 'error',
-                summary: 'Error!',
-                detail: 'There was an error uploading your file. Please try again later',
-                life: 3000
-            });
-
+        } catch (e) {
             this.uploading = false;
             this.fileUploaded = false;
+
+            if (isServerError(e)) {
+                if (e.response.data.hasOwnProperty('errors')) {
+                    const errors = e.response.data.errors;
+
+                    if (errors.hasOwnProperty('file')) {
+                        this.error = errors.file[0];
+
+                        return;
+                    }
+                }
+
+                if (e.response.data.hasOwnProperty('message')) {
+                    this.error = e.response.data.message;
+
+                    return;
+                }
+            }
+
+            this.error = 'There was an error uploading your file. Please try again later.';
         }
     }