newroco
/
searchanddisplace-core
9
0
Fork 0
Code Issues 5 Pull Requests Projects Releases Wiki Activity
Browse Source

Fix issue with handling PDF files. Fix issue with running the operations from the command line.

master
Orzu Ionut 3 years ago
parent
89ffcdd8eb
commit
68621b66d8
8 changed files with 180 additions and 49 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 66
      app/Console/Commands/RunSearchDisplace.php
  2. 6
      app/Http/Controllers/FileController.php
  3. 10
      app/SearchDisplace/Ingest/SendDocument.php
  4. 21
      app/SearchDisplace/SearchAndDisplaceFromFiles.php
  5. 1
      app/SearchDisplace/Webhooks/HandleRequest.php
  6. 63
      public/js/app.js
  7. 35
      resources/js/components/ProcessFile/ProcessFile.ts
  8. 27
      resources/js/components/ProcessFile/ProcessFile.vue

66
app/Console/Commands/RunSearchDisplace.php
View File

@ -44,13 +44,20 @@ class RunSearchDisplace extends Command
$searchers = $this->argument('filters');
$id = md5(uniqid(rand(), true));
$pathDetails = pathinfo($documentPath);
$this->storeSearchers($id, $searchers, $documentPath);
$resultedDocumentPath = $pathDetails['dirname'] . '/' . $pathDetails['filename'] . '-displaced.md';
$this->storeSearchers($id, $searchers, $resultedDocumentPath);
try {
$sendToIngest = new SendDocument();
$sendToIngest->execute($documentPath, $id);
$sendToIngest->execute($id, [
'path' => $documentPath,
'name' => $pathDetails['basename'],
'type' => $this->getFileMimeType($documentPath),
]);
$this->info('Processing document..');
$this->info('After the processing will be done the result will show up at the same path as the input.');
@ -59,11 +66,12 @@ class RunSearchDisplace extends Command
}
}
protected function storeSearchers($id, $searchers, $documentPath)
protected function storeSearchers($id, $searchers, $storeResultPath)
{
$data = [
'searchers' => $this->getSearchers($searchers),
'document_path' => $documentPath,
'searchers' => $this->getSearchers($searchers),
'document_path' => $storeResultPath,
];
$storage = Storage::disk('local');
@ -95,14 +103,58 @@ class RunSearchDisplace extends Command
return $list;
}
protected function getSearchersFromFile($path)
protected function getSearchersFromFile($argument)
{
$result = explode(':', $argument);
if (count($result) > 1) {
$path = $result[0];
$replaceWith = $result[1];
} else {
$path = $argument;
$replaceWith = '';
}
$contents = file_get_contents($path);
if ( ! $contents) {
throw new \Exception('Something went wrong when tried reading from file.');
}
return json_decode($contents);
return [
[
'content' => json_decode($contents),
'replace_with' => $replaceWith,
],
];
}
protected function getFileMimeType($file) {
if (function_exists('finfo_file')) {
$finfo = finfo_open(FILEINFO_MIME_TYPE);
$type = finfo_file($finfo, $file);
finfo_close($finfo);
} else {
require_once 'upgradephp/ext/mime.php';
$type = mime_content_type($file);
}
if (!$type || in_array($type, array('application/octet-stream', 'text/plain'))) {
$secondOpinion = exec('file -b --mime-type ' . escapeshellarg($file), $foo, $returnCode);
if ($returnCode === 0 && $secondOpinion) {
$type = $secondOpinion;
}
}
if (!$type || in_array($type, array('application/octet-stream', 'text/plain'))) {
require_once 'upgradephp/ext/mime.php';
$exifImageType = exif_imagetype($file);
if ($exifImageType !== false) {
$type = image_type_to_mime_type($exifImageType);
}
}
return $type;
}
}

6
app/Http/Controllers/FileController.php
View File

@ -26,7 +26,11 @@ class FileController extends Controller
$sendDocument = new SendDocument();
$sendDocument->execute($file->getRealPath(), $fileId);
$sendDocument->execute($fileId, [
'path' => $file->getRealPath(),
'type' => $file->getMimeType(),
'name' => $file->getClientOriginalName()
]);
return response()->json([
'id' => $fileId,

10
app/SearchDisplace/Ingest/SendDocument.php
View File

@ -13,10 +13,10 @@ class SendDocument
$this->url = env('SD_INGEST_URL') . '/ingest';
}
public function execute($documentPath, $id)
public function execute($id, $document)
{
try {
$response = $this->sendRequest($documentPath, $id);
$response = $this->sendRequest($id, $document);
if ($response['status'] === 'fail') {
$message = array_key_exists('message', $response)
@ -32,7 +32,7 @@ class SendDocument
}
}
public function sendRequest($documentPath, $id)
public function sendRequest($id, $document)
{
$client = new Client();
@ -49,7 +49,9 @@ class SendDocument
[
'name' => 'document',
'contents' => fopen($documentPath, 'r'),
'contents' => fopen($document['path'], 'r'),
'filename' => $document['name'],
'Content-type' => $document['type'],
]
],
]);

21
app/SearchDisplace/SearchAndDisplaceFromFiles.php
View File

@ -31,22 +31,27 @@ class SearchAndDisplaceFromFiles
}
$documentContent = $this->storage->get($this->documentFilePath);
$info = json_decode($this->storage->get($this->infoFilePath), true);
$searchersContent = json_decode($this->storage->get($this->infoFilePath), true);
$searchAndDisplace = new SearchAndDisplace($documentContent, $info);
$documentPath = $searchersContent['document_path'];
$searchers = $searchersContent['searchers'];
$originalDocumentPath = $info['document_path'];
$pathDetails = pathinfo($originalDocumentPath);
$this->storage->put($this->infoFilePath, json_encode($searchers[0]['content']));
$resultedDocumentPath = $pathDetails['dirname'] . '/' . $pathDetails['filename'] . '-displaced.md';
$searchAndDisplace = new SearchAndDisplace($documentContent, [
'searchers' => [
[
'key' => $this->id,
'replace_with' => $searchers[0]['replace_with'],
]
],
]);
try {
$result = $searchAndDisplace->execute();
file_put_contents($resultedDocumentPath, $result['content']);
file_put_contents($documentPath, $result['content']);
} catch (\Exception $exception) {
\Illuminate\Support\Facades\Log::info('exception: ' . $exception->getMessage());
return;
} finally {
$this->storage->delete($this->documentFilePath);

1
app/SearchDisplace/Webhooks/HandleRequest.php
View File

@ -16,7 +16,6 @@ class HandleRequest extends ProcessWebhookJob
return true;
} catch (\Exception $exception) {
return false;
}
}

63
public/js/app.js
View File

@ -3487,6 +3487,12 @@ function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try
function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; }
function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }
function _defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } }
@ -3539,13 +3545,14 @@ var ProcessFile = /*#__PURE__*/function (_Vue) {
_this.searchersFilters = []; // The list of selected filters/searchers
_this.selectedSearchers = []; //The list of expanded rows in the selected filters/searchers table
_this.selectedSearchers = {}; //The list of expanded rows in the selected filters/searchers table
_this.expandedRows = []; // The list of options applied to the searchers (for the moment, only replace_with)
_this.searchersOptions = {}; // Flag to determine whether or not we will show the diff highlights
_this.showDiffHighlight = false;
_this.newlySelectedSearchers = [];
return _this;
}
/**
@ -3608,6 +3615,24 @@ var ProcessFile = /*#__PURE__*/function (_Vue) {
} else {
this.searchersDialogVisible = !this.searchersDialogVisible;
}
if (!this.searchersDialogVisible) {
var _iterator = _createForOfIteratorHelper(this.newlySelectedSearchers),
_step;
try {
for (_iterator.s(); !(_step = _iterator.n()).done;) {
var selectedSearcher = _step.value;
this.selectedSearchers[selectedSearcher.id] = selectedSearcher;
}
} catch (err) {
_iterator.e(err);
} finally {
_iterator.f();
}
this.newlySelectedSearchers = [];
}
}
/**
* Wait for the file to be processed in ingest
@ -3672,6 +3697,11 @@ var ProcessFile = /*#__PURE__*/function (_Vue) {
value: function onSelectedSearchersReorder($event) {
this.selectedSearchers = $event.value;
}
}, {
key: "confirmDeleteProduct",
value: function confirmDeleteProduct(searcher) {
this.$delete(this.selectedSearchers, searcher.id);
}
/**
* Run the searchers
*/
@ -3690,7 +3720,7 @@ var ProcessFile = /*#__PURE__*/function (_Vue) {
this.processing = true;
this.processedFileContent = '';
searchers = [];
this.selectedSearchers.forEach(function (searcher) {
Object.values(this.selectedSearchers).forEach(function (searcher) {
searchers.push({
'key': searcher.id,
'replace_with': _this2.searchersOptions[searcher.id] || ''
@ -28432,7 +28462,7 @@ var render = function() {
staticClass: "p-datatable-sm",
attrs: {
value: _vm.searchers,
selection: _vm.selectedSearchers,
selection: _vm.newlySelectedSearchers,
dataKey: "id",
selectionMode: "multiple",
metaKeySelection: false,
@ -28449,7 +28479,7 @@ var render = function() {
_vm.searchers = $event
},
"update:selection": function($event) {
_vm.selectedSearchers = $event
_vm.newlySelectedSearchers = $event
}
}
},
@ -28476,13 +28506,13 @@ var render = function() {
{
staticClass: "p-datatable-sm",
attrs: {
value: _vm.selectedSearchers,
value: Object.values(_vm.selectedSearchers),
dataKey: "id",
expandedRows: _vm.expandedRows
},
on: {
"update:value": function($event) {
_vm.selectedSearchers = $event
_vm.$set(Object, "values(selectedSearchers)", $event)
},
"update:expandedRows": function($event) {
_vm.expandedRows = $event
@ -28554,6 +28584,27 @@ var render = function() {
_vm._v(" "),
_c("Column", { attrs: { field: "name", header: "Name" } }),
_vm._v(" "),
_c("Column", {
scopedSlots: _vm._u([
{
key: "body",
fn: function(slotProps) {
return [
_c("Button", {
staticClass: "p-button-rounded p-button-warning",
attrs: { icon: "pi pi-trash" },
on: {
click: function($event) {
return _vm.confirmDeleteProduct(slotProps.data)
}
}
})
]
}
}
])
}),
_vm._v(" "),
_c("Column", {
attrs: { expander: true, headerStyle: "width: 3rem" }
})

35
resources/js/components/ProcessFile/ProcessFile.ts
View File

@ -1,6 +1,6 @@
import marked from 'marked';
import { Vue, Component, Prop, Watch } from 'vue-property-decorator';
import { FileData } from '@/interfaces/FileData';
import {Vue, Component, Prop, Watch} from 'vue-property-decorator';
import {FileData} from '@/interfaces/FileData';
@Component
export default class ProcessFile extends Vue {
@ -31,7 +31,7 @@ export default class ProcessFile extends Vue {
private processedFileContentPreview: string = '';
private documentDiffIndexes: { [key:string]: Array<{ start: number; end: number; }>; } = {};
private documentDiffIndexes: { [key: string]: Array<{ start: number; end: number; }>; } = {};
// Flag to determine whether the text is processing or not
private processing: boolean = false;
@ -49,7 +49,7 @@ export default class ProcessFile extends Vue {
private searchersFilters: any = [];
// The list of selected filters/searchers
private selectedSearchers: Array<{ id: string; name: string; }> = [];
private selectedSearchers: any = {};
//The list of expanded rows in the selected filters/searchers table
private expandedRows: Array<{ id: string; name: string; }> = [];
@ -60,6 +60,8 @@ export default class ProcessFile extends Vue {
// Flag to determine whether or not we will show the diff highlights
private showDiffHighlight: boolean = false;
private newlySelectedSearchers: Array<{ id: string; name: string; }> = [];
/**
*
*/
@ -101,12 +103,19 @@ export default class ProcessFile extends Vue {
* @param {string} newValue The new value for the dialog visibility
*/
private toggleSearchersDialog(newValue?: boolean) {
if (typeof newValue !== 'undefined') {
this.searchersDialogVisible = newValue;
} else {
this.searchersDialogVisible = !this.searchersDialogVisible;
}
if ( ! this.searchersDialogVisible) {
for (let selectedSearcher of this.newlySelectedSearchers) {
this.selectedSearchers[selectedSearcher.id] = selectedSearcher;
}
this.newlySelectedSearchers = [];
}
}
/**
@ -140,16 +149,18 @@ export default class ProcessFile extends Vue {
}
}
/**
*
* @param $event
*/
private onSelectedSearchersReorder($event: any)
{
private onSelectedSearchersReorder($event: any) {
this.selectedSearchers = $event.value;
}
private confirmDeleteProduct(searcher: any) {
this.$delete(this.selectedSearchers, searcher.id);
}
/**
* Run the searchers
*/
@ -158,7 +169,8 @@ export default class ProcessFile extends Vue {
this.processedFileContent = '';
let searchers: Array<{ key: string; replace_with: string; }> = [];
this.selectedSearchers.forEach((searcher) => {
Object.values(this.selectedSearchers).forEach((searcher: any) => {
searchers.push({
'key': searcher.id,
'replace_with': this.searchersOptions[searcher.id] || ''
@ -180,7 +192,7 @@ export default class ProcessFile extends Vue {
private createDiffPreview() {
this.processedFileContentPreview = this.processedFileContent;
let indexes: Array<{start: number; end: number}> = [];
let indexes: Array<{ start: number; end: number }> = [];
for (let searcher in this.documentDiffIndexes) {
const searcherIndexes = this.documentDiffIndexes[searcher];
@ -220,8 +232,7 @@ export default class ProcessFile extends Vue {
* @param {boolean} oldValue
*/
@Watch('showDiffHighlight')
private onDiffHighlightChanged(newValue: boolean, oldValue: boolean): void
{
private onDiffHighlightChanged(newValue: boolean, oldValue: boolean): void {
console.log('OLD: ', oldValue);
console.log('NEW: ', newValue);
}

27
resources/js/components/ProcessFile/ProcessFile.vue
View File

@ -128,18 +128,17 @@
</div>
</div>
<Dialog
header="Available document searchers"
:visible.sync="searchersDialogVisible"
:maximizable="true"
:style="{width: '50vw'}"
:contentStyle="{overflow: 'visible'}"
id="searchers_dialog"
ref="searchers-dialog">
<Dialog header="Available document searchers"
:visible.sync="searchersDialogVisible"
:maximizable="true"
:style="{width: '50vw'}"
:contentStyle="{overflow: 'visible'}"
id="searchers_dialog"
ref="searchers-dialog">
<DataTable
:value.sync="searchers"
:selection.sync="selectedSearchers"
:selection.sync="newlySelectedSearchers"
dataKey="id"
selectionMode="multiple"
class="p-datatable-sm"
@ -166,7 +165,7 @@
</Dialog>
<DataTable
:value.sync="selectedSearchers"
:value.sync="Object.values(selectedSearchers)"
dataKey="id"
class="p-datatable-sm"
:expandedRows.sync="expandedRows"
@ -176,6 +175,14 @@
<Column field="name" header="Name"></Column>
<Column>
<template #body="slotProps">
<Button icon="pi pi-trash"
class="p-button-rounded p-button-warning"
@click="confirmDeleteProduct(slotProps.data)"/>
</template>
</Column>
<Column :expander="true" headerStyle="width: 3rem"/>
<template #expansion="slotProps">

Write Preview
Loading…
Cancel
Save