Browse Source

Initial commit. Process files and send response via webhook

hidden_tags_with_bookmarks
Orzu Ionut 1 year ago
commit
e45e5d27bb
  1. 15
      .editorconfig
  2. 20
      .env.example
  3. 5
      .gitattributes
  4. 16
      .gitignore
  5. 13
      .styleci.yml
  6. 491
      README.md
  7. 76
      app/Console/Commands/DeployWorker.php
  8. 57
      app/Console/Commands/TestMachine.php
  9. 46
      app/Console/Kernel.php
  10. 71
      app/Exceptions/Handler.php
  11. 20
      app/Helpers/array.php
  12. 13
      app/Http/Controllers/Controller.php
  13. 31
      app/Http/Controllers/IngestController.php
  14. 82
      app/Http/Kernel.php
  15. 21
      app/Http/Middleware/Authenticate.php
  16. 17
      app/Http/Middleware/CheckForMaintenanceMode.php
  17. 17
      app/Http/Middleware/EncryptCookies.php
  18. 27
      app/Http/Middleware/RedirectIfAuthenticated.php
  19. 18
      app/Http/Middleware/TrimStrings.php
  20. 23
      app/Http/Middleware/TrustProxies.php
  21. 24
      app/Http/Middleware/VerifyCsrfToken.php
  22. 190
      app/Ingest/Convertor.php
  23. 56
      app/Ingest/DocumentHandler.php
  24. 45
      app/Ingest/MDConvertor.php
  25. 172
      app/Jobs/IngestDocuments.php
  26. 101
      app/Jobs/SendToCore.php
  27. 30
      app/Listeners/Test.php
  28. 19
      app/Listeners/WebhookFailedToSendToCoreListener.php
  29. 19
      app/Listeners/WebhookSuccessfullySentToCoreListener.php
  30. 17
      app/Parser/DocxParser/Footer.php
  31. 41
      app/Parser/DocxParser/Footnote.php
  32. 11
      app/Parser/DocxParser/Header.php
  33. 26
      app/Parser/DocxParser/Link.php
  34. 77
      app/Parser/DocxParser/ListItemRun.php
  35. 11
      app/Parser/DocxParser/PageBreak.php
  36. 269
      app/Parser/DocxParser/ParseDocx.php
  37. 32
      app/Parser/DocxParser/PreserveText.php
  38. 41
      app/Parser/DocxParser/Section.php
  39. 35
      app/Parser/DocxParser/Table.php
  40. 41
      app/Parser/DocxParser/Table/Cell.php
  41. 41
      app/Parser/DocxParser/Table/Row.php
  42. 147
      app/Parser/DocxParser/Text.php
  43. 17
      app/Parser/DocxParser/TextBreak.php
  44. 74
      app/Parser/DocxParser/TextRun.php
  45. 72
      app/Parser/DocxParser/Title.php
  46. 117
      app/Parser/DocxParser/Traits/Helper.php
  47. 527
      app/Parser/HtmlParser/ParseHtml.php
  48. 670
      app/Parser/ParseHtmlArray.php
  49. 747
      app/Parser/ParseTextArray.php
  50. 406
      app/Parser/ParseXml.php
  51. 28
      app/Providers/AppServiceProvider.php
  52. 30
      app/Providers/AuthServiceProvider.php
  53. 21
      app/Providers/BroadcastServiceProvider.php
  54. 41
      app/Providers/EventServiceProvider.php
  55. 80
      app/Providers/RouteServiceProvider.php
  56. 53
      artisan
  57. 55
      bootstrap/app.php
  58. 2
      bootstrap/cache/.gitignore
  59. 69
      composer.json
  60. 5876
      composer.lock
  61. 231
      config/app.php
  62. 117
      config/auth.php
  63. 59
      config/broadcasting.php
  64. 103
      config/cache.php
  65. 147
      config/database.php
  66. 79
      config/filesystems.php
  67. 52
      config/hashing.php
  68. 104
      config/logging.php
  69. 136
      config/mail.php
  70. 88
      config/queue.php
  71. 33
      config/services.php
  72. 199
      config/session.php
  73. 36
      config/view.php
  74. 61
      config/webhook-server.php
  75. 2
      database/.gitignore
  76. 35
      database/migrations/2019_08_19_000000_create_failed_jobs_table.php
  77. 16
      database/seeds/DatabaseSeeder.php
  78. 24306
      get-pip.py
  79. 21
      package.json
  80. 37
      phpunit.xml
  81. 22
      public/.htaccess
  82. BIN
      public/favicon.ico
  83. 60
      public/index.php
  84. 2
      public/robots.txt
  85. 28
      public/web.config
  86. 1
      resources/js/app.js
  87. 28
      resources/js/bootstrap.js
  88. 19
      resources/lang/en/auth.php
  89. 19
      resources/lang/en/pagination.php
  90. 22
      resources/lang/en/passwords.php
  91. 151
      resources/lang/en/validation.php
  92. 1
      resources/sass/app.scss
  93. 5
      resources/views/errors/401.blade.php
  94. 5
      resources/views/errors/403.blade.php
  95. 4
      resources/views/errors/404.blade.php
  96. 4
      resources/views/errors/405.blade.php
  97. 5
      resources/views/errors/419.blade.php
  98. 6
      resources/views/errors/429.blade.php
  99. 5
      resources/views/errors/500.blade.php
  100. 5
      resources/views/errors/503.blade.php

15
.editorconfig

@ -0,0 +1,15 @@
root = true
[*]
charset = utf-8
end_of_line = lf
insert_final_newline = true
indent_style = space
indent_size = 4
trim_trailing_whitespace = true
[*.md]
trim_trailing_whitespace = false
[*.{yml,yaml}]
indent_size = 2

20
.env.example

@ -0,0 +1,20 @@
APP_NAME=Laravel
APP_ENV=local
APP_KEY=
APP_DEBUG=true
APP_URL=http://localhost
LOG_CHANNEL=stack
BROADCAST_DRIVER=log
CACHE_DRIVER=file
QUEUE_CONNECTION=sync
SESSION_DRIVER=file
SESSION_LIFETIME=120
REDIS_HOST=127.0.0.1
REDIS_PASSWORD=null
REDIS_PORT=6379
WEBHOOK_CORE_URL=
WEBHOOK_CORE_SECRET=

5
.gitattributes

@ -0,0 +1,5 @@
* text=auto
*.css linguist-vendored
*.scss linguist-vendored
*.js linguist-vendored
CHANGELOG.md export-ignore

16
.gitignore

@ -0,0 +1,16 @@
/node_modules
/public/hot
/public/storage
/storage/*.key
/vendor
.idea
/storage/app/ingest_conversions/*
/storage/app/ingest_queue/*
.env
.env.backup
.phpunit.result.cache
Homestead.json
Homestead.yaml
npm-debug.log
yarn-error.log
.idea

13
.styleci.yml

@ -0,0 +1,13 @@
php:
preset: laravel
disabled:
- unused_use
finder:
not-name:
- index.php
- server.php
js:
finder:
not-name:
- webpack.mix.js
css: true

491
README.md

@ -0,0 +1,491 @@
## About S&D Ingest
S&D INGEST it's the module that receives row files in different formats and send's them to any module after the file's are being processed.
## :cyclone: Server Requirements:
- php7.4 [https://www.php.net] [LICENSE](https://www.php.net/license/index.php)
- apache [https://httpd.apache.org] [LICENSE](hhttps://www.apache.org/licenses/LICENSE-2.0)
- redis [https://redis.io] [LICENSE](https://redislabs.com/legal/licenses/)
- postgresql-server [https://www.postgresql.org] [LICENSE](https://tldrlegal.com/license/postgresql-license-(postgresql))
- supervisor [http://supervisord.org] [LICENSE](https://github.com/Supervisor/supervisor/blob/master/LICENSES.txt)
- libraoffice [https://www.libreoffice.org] [LICENSE](https://www.libreoffice.org/about-us/licenses)
- python [https://www.python.org/] [LICENSE](https://www.python.org/download/releases/2.7/license/)
- pdftotext [https://github.com/jalan/pdftotext] [LICENSE](https://github.com/jalan/pdftotext/blob/master/LICENSE)
## :zap: Build with:
- Laravel Framework ^6.2
## :rocket: Installation
```bash
apt-get update
apt-get install software-properies-common
add-apt-repository ppa:deadsnakes/ppa
apt-get install supervisor python3.8 python3.8-dev
supervisorctl restart all
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
python get-pip.py
rm -rf get-pip.py
apt install libpoppler-cpp-dev
pip install --upgrade pip
pip install pdftotext supervisor
systemctl enable supervisor
php artisan queue:deploy-supervisor
systemctl restart supervisor
composer install
npm install
cp .env.example .env
php artisan key:generate
sudo -u postgres psql
postgres=# create database mydb;
postgres=# create user myuser with encrypted password 'mypass';
postgres=# grant all privileges on database mydb to myuser;
#update the .env with the current postgres credentials
sudo mkdir /var/log/amqp
sudo mkdir /var/log/queue
php artisan migrate
php artisan queue:deploy-supervisor
supervisorctl start all
```
## Local Usage
```python
php artisan serve
php artisan queue:work
```
## PHP Packages
- fideloper/proxy [LICENSE](https://github.com/fideloper/TrustedProxy/blob/master/LICENSE.md)
- laravel/framework [LICENSE](https://github.com/laravel/framework/blob/7.x/LICENSE.md)
- laravel/tinker [LICENSE](https://github.com/laravel/tinker/blob/2.x/LICENSE.md)
- phpoffice/phpword [LICENSE](https://github.com/PHPOffice/PHPWord/blob/0.17.0/LICENSE)
- predis/predis [LICENSE](https://github.com/php-enqueue/amqp-bunny/blob/master/LICENSE)
- spatie/laravel-webhook-server [LICENSE](https://github.com/spatie/laravel-webhook-server/blob/master/LICENSE.md)
## Current running process
- [DOC,DOCX,RTF etc..] are first being converted to docx and then converted to .txt using https://www.libreoffice.org
- [PDF] files are converted to .txt
- The resulting .txt file is processed using our own logic/alogorithm and clause breaking point to an array that looks similar to this:
```json
{
"content": "Definitions and Interpretation",
"spaces": 4,
"numbering": "1.",
"children": [
{
"content": "In this Agreement, the following expressions shall have the meanings set opposite them, unless inconsistent with the context or otherwise specified:",
"spaces": 8,
"numbering": "1.1",
"children": [
{
"content": "“Agreement” this agreement including all schedules, appendices and exhibits attached herein;",
"spaces": 0
},
{
"content": "“Associated Company” any company which is listed in Schedule 2 (as may be amended from time to time in writing) and which is in relation to either party its Parent undertaking or its subsidiary undertaking or a subsidiary undertaking of its Parent undertaking or any other person controlled by it or under the same control (where “control” is to be construed in accordance with section 1124 of the Corporation Tax Act 2010) whether direct or indirect. “Parent undertaking” shall have the meaning attributed thereto in Section 1162 of the Companies Act 2006;",
"spaces": 0
},
{
"content": "“Commencement Date” [TBC] “Confidential Information” collectively and individually, all or any document or information of any nature in any format, including oral, written or electronic form relating to either party or their Associated Companies’ or either of their businesses, including technology, Customers, Customer Information, supplier, employees, finances, data, products, services, trade secrets, processes, designs, drawings, diagrams, plans, specifications, formulae, testing procedures, computer software, reports, investigative studies, manuals, assets, costs, prices, marketing opportunities, proprietary information, Know-how, the terms of this Agreement and any other information or material relating to the information described above which (i) is disclosed by either party (or by any individual or legal entity acting in their name or on their behalf, including employees, consultants, sub-contractors, advisors of any kind and agents) or (ii) which comes to the attention of either party (or any individual or legal entity acting in their name or on their behalf, including employees, consultants, sub-contractors, advisors of any kind and agents) during the course of the carrying out of the rights or obligations under this Agreement;",
"spaces": 0
},
{
"content": "“Customers” customers of {P1_Name} and/or its Associated Companies from time to time who owe a Debt to {P1_Name};",
"spaces": 0
},
{
"content": "“Customer Data” any information given by the Customer directly to the {P2_Name} or its personnel;",
"spaces": 0
},
{
"content": "“Customer Information” any Customers’ personal information supplied to the {P2_Name} by or on behalf of {P1_Name} during the performance of this Agreement, including Personal Data, but excluding Customer Data;",
"spaces": 0
},
{
"content": "“Data Protection Legislation” all applicable legislation concerning the protection of individuals with regard to the processing of Personal Data and the free movement of such data including the Data Protection Act 1998 and any regulations made under such legislation and any relevant codes of practice and guidance notes issued from time to time by the Information Commissioner (or its successor);",
"spaces": 0
},
{
"content": "“Debt” any monies owed by the Customer to {P1_Name} which have remained unpaid by the Customer contrary to the terms and conditions between {P1_Name} and the Customer governing the repayment of sums owed;",
"spaces": 0
},
{
"content": "“Debt Management Plan” a plan outlined by the {P2_Name} and agreed by the Customer which details the amount and frequency of payments to be made to each of the Customer’s creditors;",
"spaces": 0
},
{
"content": "“Disbursement” total amount remitted to {P1_Name} on a monthly basis by the {P2_Name} for application to the Customer’s account in respect of their Offer;",
"spaces": 0
},
{
"content": "“European Economic Area” the European Economic Area comprising of the following countries as at the Commencement Date: Austria; Belgium; Bulgaria; Cyprus; the Czech Republic; Denmark; Estonia; Finland; France; Germany; Greece; Hungary; Ireland; Italy; Latvia; Lithuania; Luxembourg; Malta; the Netherlands; Poland; Portugal; Romania; Slovakia; Slovenia; Spain; Sweden; the United Kingdom; Iceland; Liechtenstein and Norway, as amended from time to time;",
"spaces": 0
},
{
"content": "“EU Model Terms” the set of model contractual clauses which the Information Commissioner has authorised for use by Data Controllers (as such term is defined in the Data Protection Legislation) established in the European Union where there is a transfer of Personal Data to Data Processors (as such term is defined in the Data Protection Legislation) outside of the European Economic Area;",
"spaces": 0
},
{
"content": "“Facility” the {P2_Name} site authorised by {P1_Name} where the processing and/or storage of Personal Data supplied by {P1_Name} pursuant to this Agreement takes place. For the purposes of this Agreement that site shall be located at {P1_Reg} or such other place as may be notified in writing to {P1_Name} from time to time;",
"spaces": 0
},
{
"content": "“Force Majeure” any acts, events, omissions or accidents beyond the reasonable control of either Party, including but not limited to acts of God, extreme adverse weather conditions or natural disaster, war, threat of or preparation for war, armed conflict, imposition of sanctions, embargo, breaking off of diplomatic relations or similar actions, terrorist attack, civil war, civil commotion or riots, nuclear, chemical or biological contamination or sonic boom, compliance with any law, regulation or directive, fire, explosion or accidental damage, failure of plant machinery, machinery, computers or vehicles;",
"spaces": 0
},
{
"content": "“Information Commissioner” the independent authority in the UK (or its successor body) which regulates information rights;",
"spaces": 0
},
{
"content": "“Initial Period” three (3) years from the Commencement Date;",
"spaces": 0
},
{
"content": "“Lending Code” a voluntary code of practice (enforced by the Lending Standards Board) which sets standards for financial institutions and provides consumers with protection and explanation on how such institutions are expected to deal with them day-to-day and in times of financial difficulties;",
"spaces": 0
},
{
"content": "“Notification” written notification from the {P2_Name} to {P1_Name} that it has obtained Permission;",
"spaces": 0
},
{
"content": "“Offer” a statement of proposed amount to be repaid by the Customer to {P1_Name} in respect of the Customer’s Debt including instalment plans;",
"spaces": 0
},
{
"content": "“Payment Break” instance where the Customer fails to make an agreed repayment to the {P2_Name} for payment to their creditors;",
"spaces": 0
},
{
"content": "“Permission” written confirmation (which may be confirmation by email or other electronic means) from the Customer to the {P2_Name} that they are appointing the {P2_Name} to act on the Customer’s behalf in the management of the Customer’s Debt and authorising the {P2_Name} to negotiate payment terms with {P1_Name} in respect of the Customer’s Debt and authorising the {P2_Name} to have access to Customer Information;",
"spaces": 0
},
{
"content": "“Personal Data” personal data as defined in the Data Protection Legislation;",
"spaces": 0
},
{
"content": "“Regulatory Authorities” any body who, from time to time, has competent rule-making, investigatory and/or enforcement powers in relation to the business of {P1_Name} and/or its Associated Companies, including, without limitation, the Financial Conduct Authority, the Consumer Financial Protection Bureau, the Office of Fair Trading, the Information Commissioner’s Office, the Lending Standards Board, UK and US Government departments and organisations, the Office of the Comptroller of Currency, the Federal Reserve and other governmental or non-governmental regulatory authorities in the UK, US or other competent jurisdictions;",
"spaces": 0
},
{
"content": "“Regulatory Requirements”",
"spaces": 0
},
{
"content": "(a) all applicable laws, statutes, regulations, ordinances or subordinate legislation in force from time to time to which this Agreement or a party is subject;",
"spaces": 4
},
{
"content": "(b) the common law as applicable to the parties from time to time;",
"spaces": 4
},
{
"content": "(c) all binding court orders, judgements or decrees;",
"spaces": 4
},
{
"content": "all applicable directives, policies, rules, orders, code of conduct or practice or applicable guidance (including the Lending Code and the Financial Conduct Authority TCF principles that are binding on a party and that are made or given by any government, an agency thereof, any Regulatory Authority or other regulatory authority, including in the case of the {P2_Name}, laws and rules imposed by local regulatory authorities in the country where it is located;",
"spaces": 0
},
{
"content": "“Working Day” any day on which banks in London are open for the transaction of normal business excluding Saturdays, Sundays and bank and public holidays in England and Wales.",
"spaces": 0
}
]
},
{
"content": "In this Agreement:",
"spaces": 8,
"numbering": "1.2",
"children": [
{
"content": "references to Recitals, Clauses and Schedules and their sub-divisions are to the Recitals to, Clauses of and Schedules to this Agreement and their sub-divisions respectively, unless specified otherwise;",
"spaces": 12,
"numbering": "1.2.1"
},
{
"content": "the index and headings are included for convenience only and shall not affect the construction or interpretation of this Agreement;",
"spaces": 12,
"numbering": "1.2.2"
},
{
"content": "words importing gender include the other gender and the singular includes the plural and vice versa;",
"spaces": 12,
"numbering": "1.2.3"
},
{
"content": "references to persons include individuals, bodies corporate, firms, unincorporated associations and governmental, semi-governmental and local authorities or agencies;",
"spaces": 12,
"numbering": "1.2.4"
},
{
"content": "references to the words “include”, “including”, “in particular” or similar words or expressions will be construed without limitation and accordingly will not limit the words preceding or following them;",
"spaces": 12,
"numbering": "1.2.5"
},
{
"content": "where expressions used in this Agreement are not specifically defined and are capable of having a special meaning according to the usage or custom of the card services sector or banking services sector, such expressions are to be interpreted accordingly. Any meaning given in this Agreement to a defined term shall prevail over such other special meaning;",
"spaces": 12,
"numbering": "1.2.6"
},
{
"content": "references to a “party” or “parties” will mean either {P1_Name} and/or the {P2_Name} as the context requires and references to a third party will mean any person other than the parties;",
"spaces": 12,
"numbering": "1.2.7"
},
{
"content": "except where expressly stated otherwise, references to any statute, legislation, code of practice or other regulation will include any sub-ordinate legislation and any equivalent regulation in any relevant jurisdiction, as amended, modified, consolidated, re-enacted and/or replaced and in force from time to time;",
"spaces": 12,
"numbering": "1.2.8"
},
{
"content": "any negative obligation imposed on any party shall be construed as if it were also an obligation not to permit or suffer the act or thing in question and any positive obligation imposed on any party shall be construed as if it were also an obligation to procure that the act or thing in question be done;",
"spaces": 12,
"numbering": "1.2.9"
},
{
"content": "the Schedules and Appendices (if any) form part of this Agreement and shall be construed and have the same full force and effect as if expressly set out in the body of this Agreement. To the extent only of any conflict or inconsistency between the Clauses, Schedules and Appendices (if any), the Clauses will prevail and the order of precedence will be as follows:",
"spaces": 12,
"numbering": "1.2.10"
},
{
"content": "1 the provisions of the Clauses;",
"spaces": 16,
"numbering": "1.2.10"
},
{
"content": "2 the provisions of the Schedules; and",
"spaces": 16,
"numbering": "1.2.10"
},
{
"content": "3 the provisions of the Appendices.",
"spaces": 16,
"numbering": "1.2.10"
}
]
}
]
},
{
"content": "Obligations of the {P2_Name}",
"spaces": 4,
"numbering": "2.",
"children": [
{
"content": "The {P2_Name} shall obtain the Permission from the Customer before proceeding with the Debt Management Plan.",
"spaces": 8,
"numbering": "2.1"
},
{
"content": "Subject at all times to the {P2_Name} being in receipt of the applicable Permission, {P2_Name} shall provide the corresponding Notification to {P1_Name} before or at the time of making the first Offer to {P1_Name}. In the absence of such Permission or Notification {P1_Name} shall not be obliged to provide any Customer Information to the {P2_Name}.",
"spaces": 8,
"numbering": "2.2"
},
{
"content": "{P1_Name} may request, and the {P2_Name} shall provide, any Permission to {P1_Name} within two (2) Working Days of such request by {P1_Name} to enable {P1_Name} to verify the Permissions stated in the Notifications provided that in the event that {P1_Name} requests ten (10) or more Permissions in any 12 hour period then the {P2_Name} shall provide such Permissions as promptly as is reasonably possible.",
"spaces": 8,
"numbering": "2.3"
},
{
"content": "Any delay or failure by the {P2_Name} to comply with Clause 2.3 shall be deemed a material breach of this Agreement and the provisions of clause 9.3 shall apply.",
"spaces": 8,
"numbering": "2.4"
},
{
"content": "Subject to Clause 2.1 and in accordance with the Debt Management Plan, the {P2_Name} shall make an Offer to {P1_Name} for the repayment of the Debt detailing the amount and frequency of proposed payments. Such Offer will be made in accordance with the Lending Code guidelines and based upon the principle of equitable distribution of available income (after priority payments) in line with the amount outstanding to each creditor.",
"spaces": 8,
"numbering": "2.5"
},
{
"content": "Upon receipt of the Offer from the {P2_Name} {P1_Name} may either;",
"spaces": 8,
"numbering": "2.6",
"children": [
{
"content": "accept the Offer; or",
"spaces": 12,
"numbering": "2.6.1"
},
{
"content": "reject the Offer where it considers the offer to be unreasonable by written notice to the {P2_Name}.",
"spaces": 12,
"numbering": "2.6.2"
}
]
},
{
"content": "In the event that {P1_Name} accepts an Offer, then the {P2_Name} shall arrange for the Disbursement to be repaid to {P1_Name} in accordance with the Offer within five (5) Working Days of receipt by the {P2_Name} of cleared funds from the Customer.",
"spaces": 8,
"numbering": "2.7"
},
{
"content": "In the event that {P1_Name} rejects the Offer, then the {P2_Name} shall review the Debt Management Plan and the {P2_Name} may make a new Offer to {P1_Name}.",
"spaces": 8,
"numbering": "2.8"
},
{
"content": "For the avoidance of doubt nothing in this Agreement constitutes an obligation on {P1_Name} to accept any unreasonable Offer made by the {P2_Name}.",
"spaces": 8,
"numbering": "2.9"
},
{
"content": "The {P2_Name} shall notify {P1_Name} in writing as soon as reasonably possible:",
"spaces": 8,
"numbering": "2.10",
"children": [
{
"content": "upon becoming aware of any withdrawal of a Permission or any amendment thereto made by a Customer; and",
"spaces": 12,
"numbering": "2.10.1"
},
{
"content": "of any circumstance or event which is reasonably likely to materially affect the {P2_Name}’s ability to comply with its obligations under this Agreement.",
"spaces": 12,
"numbering": "2.10.2"
}
]
},
{
"content": "Failure by the {P2_Name} to notify {P1_Name} pursuant to Clause 2.10.1 shall be deemed a material breach of this Agreement and the provisions of clause 9.3 shall apply.",
"spaces": 8,
"numbering": "2.11"
},
{
"content": "The {P2_Name} shall;",
"spaces": 8,
"numbering": "2.12",
"children": [
{
"content": "at all times act in accordance with and subject to any limitations set out in (i) the Permission and (ii) the requirements of this Agreement;",
"spaces": 12,
"numbering": "2.12.1"
},
{
"content": "comply with the reporting and review requirements set out in Schedule I.",
"spaces": 12,
"numbering": "2.12.2"
},
{
"content": "be at all times courteous and business like in its contact with the Customers;",
"spaces": 12,
"numbering": "2.12.3"
},
{
"content": "use its reasonable commercial endeavours to comply with any reasonable and lawful directions, orders and instructions which {P1_Name} may from time to time give to it in accordance with or to give effect to the provisions of this Agreement;",
"spaces": 12,
"numbering": "2.12.4"
},
{
"content": "identify, procure and keep in force all permits, certificates, licences, approvals, authorisations and consents which may be necessary in connection with the performance of its obligations under this Agreement;",
"spaces": 12,
"numbering": "2.12.5"
},
{
"content": "in performing its obligations under this Agreement, ensure that it is knowledgeable about and shall continue to be knowledgable about all Regulatory Requirements and that it shall comply with all Regulatory Requirements and (i) maintain evidence of its compliance with Regulatory Requirements, (ii) take all necessary steps required to comply with such Regulatory Requirements promptly upon becoming aware it is not so complying, and (iii) take all necessary steps to remedy any previous breaches of such Regulatory Requirements;",
"spaces": 12,
"numbering": "2.12.6"
},
{
"content": "where permitted to do so, promptly notify {P1_Name} in the event that a regulatory body who regulates {P1_Name} or the {P2_Name} conducts an audit or investigation of the {P2_Name} and disclose to {P1_Name} (subject always to the provisions of confidentiality set out at Clause 7) details of any adverse regulatory findings; and",
"spaces": 12,
"numbering": "2.12.7"
},
{
"content": "co-operate with {P1_Name} and assist them in their dealings with Regulatory Authorities to the extent reasonably required in relation to this Agreement including implementing such measures as are reasonably necessary and appropriate to effect compliance with Regulatory Requirements.",
"spaces": 12,
"numbering": "2.12.8"
}
]
},
{
"content": "{P1_Name} acknowledges and accepts that the {P2_Name} may give advice and assistance and provide services and products beyond the scope of the Debt Management Plan to Customers and that the {P2_Name} will not disclose any Customer Data to {P1_Name} without the Customer’s prior consent (which the {P2_Name} is under no obligation to seek).",
"spaces": 8,
"numbering": "2.13"
},
{
"content": "Any failure or inability of a Customer to agree to or comply with a Debt Management Plan or any other advice or assistance given by the {P2_Name} pursuant to this Agreement shall not cause the {P2_Name} to be in breach of the terms of this Agreement and shall not prevent the {P2_Name} from providing advice for debt negotiations, counselling and management solutions outside of the Services.",
"spaces": 8,
"numbering": "2.14"
},
{
"content": "The Parties acknowledge that the {P2_Name} is not acting as an agent of {P1_Name} and that it is not a debt collection agent of {P1_Name}.",
"spaces": 8,
"numbering": "2.15"
}
]
},
{
"content": "Rights and Obligations of {P1_Name}",
"spaces": 4,
"numbering": "3.",
"children": [
{
"content": "During the term of this Agreement {P1_Name} shall provide such information and assistance as is reasonably required for the {P2_Name} to perform its obligations under this Agreement.",
"spaces": 8,
"numbering": "2.16"
},
{
"content": "For the period of six months from termination or expiry of this Agreement {P1_Name} shall not, without the prior written agreement of the {P2_Name}, employ or engage on any basis or offer such employment or engagement to any of the {P2_Name}’s personnel provided that employment or engagement of any member of the {P2_Name}’s personnel pursuant to a bona fide recruitment campaign shall not be a breach of this clause.",
"spaces": 8,
"numbering": "2.17"
},
{
"content": "{P1_Name} represents and warrants that:",
"spaces": 8,
"numbering": "2.18",
"children": [
{
"content": "it has the requisite power and authority required by any applicable law or otherwise to enter into this Agreement and to carry out the obligations contemplated by the Agreement reliably and professionally and that the execution and performance of this Agreement has been duly authorised by the required corporate action by {P1_Name};",
"spaces": 12,
"numbering": "2.18.1"
},
{
"content": "it has and shall maintain during the continuance of this Agreement all necessary rights, licences and consents necessary to provide the Customer Information to the {P2_Name} and to perform its obligations under this Agreement.",
"spaces": 12,
"numbering": "2.18.2"
}
]
},
{
"content": "If {P1_Name} notifies the {P2_Name} in writing that amendments are required to be made to this Agreement (including any Schedule hereto) to ensure {P1_Name}’s compliance with its obligations to a Regulatory Authority and/or any Regulatory Requirements (including changes required in order to comply with any rules or guidance (including guidance as to interpretation of such rules) issued or published by or on behalf of such Regulatory Authorities or coming into force from time to time), the {P2_Name} shall be obliged to make such amendments as soon as reasonably practicable and in shall use reasonable commercial endeavours to ensure that such changes are made in sufficient time so as to ensure that {P1_Name} is complying with such obligations.",
"spaces": 8,
"numbering": "2.19"
},
{
"content": "In the event the {P2_Name} is unable to comply with any amendments as notified to it by {P1_Name} pursuant to Clause 3.1 or fails to comply within a reasonable time then {P1_Name} may terminate this Agreement immediately.",
"spaces": 8,
"numbering": "2.20"
},
{
"content": "{P1_Name} shall comply with the reporting and review requirements set out in Schedule I.",
"spaces": 8,
"numbering": "2.21"
},
{
"content": "Notwithstanding Clause 2.21 above {P1_Name} shall not make any changes to a Customer’s {P1_Name} account without direct contact with the Customer. For the avoidance of doubt the Permission shall only relate to the provision of information regarding a Customer’s {P1_Name} account.",
"spaces": 8,
"numbering": "2.22"
}
]
},
{
"content": "Conditions",
"spaces": 4,
"numbering": "4.",
"children": [
{
"content": "It is a condition of this Agreement that each party is entitled to enter into this Agreement and to perform its obligations set out herein.",
"spaces": 8,
"numbering": "2.23"
}
]
},
```

76
app/Console/Commands/DeployWorker.php

@ -0,0 +1,76 @@
<?php
namespace App\Console\Commands;
use Exception;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Storage;
class DeployWorker extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'queue:deploy-supervisor';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Adds the supervisorctl config file for laravel queues. Must be ran as root!';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$workerName = 'queue-worker-'.str_replace(' ', '-', strtolower(env('APP_NAME'))).'-'.str_replace(' ', '-', strtolower(env('APP_ENV')));
$workerFile = $workerName.'.conf';
try {
Storage::disk('supervisor')->put($workerFile, '[program:'.$workerName.']
process_name=%(program_name)s_%(process_num)02d');
Storage::disk('supervisor')->append($workerFile, 'command=php '.base_path().'/artisan queue:work');
Storage::disk('supervisor')->append($workerFile, 'autostart=true
autorestart=true
user=www-data
numprocs=1
redirect_stderr=true
stdout_logfile=/var/log/queue/'.$workerName.'.log');
} catch (Exception $e) {
$this->info('supervisor script failed to install. Did you install supervisor and are you running this script as root?');
return;
}
$this->info('supervisor script installed');
try {
exec('sudo supervisorctl reread');
exec('sudo supervisorctl update');
exec('sudo supervisorctl stop '.$workerName.':*');//in case it's already started
exec('sudo supervisorctl start '.$workerName.':*');
} catch (Exception $e) {
$this->info('failed to start queue worker');
return;
}
$this->info('queue worker started');
}
}

57
app/Console/Commands/TestMachine.php
File diff suppressed because it is too large
View File

46
app/Console/Kernel.php

@ -0,0 +1,46 @@
<?php
namespace App\Console;
use App\Console\Commands\DeployWorker;
use App\Console\Commands\TestMachine;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
class Kernel extends ConsoleKernel
{
/**
* The Artisan commands provided by your application.
*
* @var array
*/
protected $commands = [
DeployWorker::class,
TestMachine::class
//
];
/**
* Define the application's command schedule.
*
* @param \Illuminate\Console\Scheduling\Schedule $schedule
* @return void
*/
protected function schedule(Schedule $schedule)
{
// $schedule->command('inspire')
// ->hourly();
}
/**
* Register the commands for the application.
*
* @return void
*/
protected function commands()
{
$this->load(__DIR__.'/Commands');
require base_path('routes/console.php');
}
}

71
app/Exceptions/Handler.php

@ -0,0 +1,71 @@
<?php
namespace App\Exceptions;
use Exception;
use Illuminate\Foundation\Exceptions\Handler as ExceptionHandler;
use Symfony\Component\HttpKernel\Exception\HttpException;
use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
class Handler extends ExceptionHandler
{
/**
* A list of the exception types that are not reported.
*
* @var array
*/
protected $dontReport = [//
];
/**
* A list of the inputs that are never flashed for validation exceptions.
*
* @var array
*/
protected $dontFlash = [
'password',
'password_confirmation',
];
/**
* Report or log an exception.
*
* @param \Exception $exception
*
* @return void
*
* @throws \Exception
*/
public function report(Exception $exception)
{
parent::report($exception);
}
/**
* Render an exception into an HTTP response.
*
* @param \Illuminate\Http\Request $request
* @param \Exception $exception
*
* @return \Symfony\Component\HttpFoundation\Response
*
* @throws \Exception
*/
public function render($request, Exception $exception)
{
if ($this->isHttpException($exception))
{
$statusCode = $exception->getStatusCode();
return response()->view("errors.".$statusCode, ['code'=>$statusCode], $statusCode);
}
if ($exception instanceof HttpException) {
return response()->view('errors::404', [], 404);
}
return parent::render($request, $exception);
}
}

20
app/Helpers/array.php

@ -0,0 +1,20 @@
<?php
/**
* Flatten an array
*
* @param array $array
*
* @return array
*/
if (! function_exists('array_flatten')) {
function array_flatten(array $array)
{
$result_array = [];
foreach ($array as $item) {
$result_array = array_merge($result_array, $item);
}
return $result_array;
}
}

13
app/Http/Controllers/Controller.php

@ -0,0 +1,13 @@
<?php
namespace App\Http\Controllers;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Illuminate\Foundation\Validation\ValidatesRequests;
use Illuminate\Routing\Controller as BaseController;
class Controller extends BaseController
{
use AuthorizesRequests, DispatchesJobs, ValidatesRequests;
}

31
app/Http/Controllers/IngestController.php

@ -0,0 +1,31 @@
<?php
namespace App\Http\Controllers;
use App\Ingest\DocumentHandler;
class IngestController extends Controller
{
public function store()
{
request()->validate([
'id' => 'required',
'document' => 'required|file',
]);
try {
$handler = new DocumentHandler(request()->get('id'), request()->file('document'));
$handler->handle();
return response()->json([
'status' => 'processing',
]);
} catch (\Exception $exception) {
return response()->json([
'status' => 'error',
'message' => $exception->getMessage(),
], 400);
}
}
}

82
app/Http/Kernel.php

@ -0,0 +1,82 @@
<?php
namespace App\Http;
use Illuminate\Foundation\Http\Kernel as HttpKernel;
class Kernel extends HttpKernel
{
/**
* The application's global HTTP middleware stack.
*
* These middleware are run during every request to your application.
*
* @var array
*/
protected $middleware = [
\App\Http\Middleware\TrustProxies::class,
\App\Http\Middleware\CheckForMaintenanceMode::class,
\Illuminate\Foundation\Http\Middleware\ValidatePostSize::class,
\App\Http\Middleware\TrimStrings::class,
\Illuminate\Foundation\Http\Middleware\ConvertEmptyStringsToNull::class,
];
/**
* The application's route middleware groups.
*
* @var array
*/
protected $middlewareGroups = [
'web' => [
\App\Http\Middleware\EncryptCookies::class,
\Illuminate\Cookie\Middleware\AddQueuedCookiesToResponse::class,
\Illuminate\Session\Middleware\StartSession::class,
// \Illuminate\Session\Middleware\AuthenticateSession::class,
\Illuminate\View\Middleware\ShareErrorsFromSession::class,
\App\Http\Middleware\VerifyCsrfToken::class,
\Illuminate\Routing\Middleware\SubstituteBindings::class,
],
'api' => [
'throttle:60,1',
\Illuminate\Routing\Middleware\SubstituteBindings::class,
],
];
/**
* The application's route middleware.
*
* These middleware may be assigned to groups or used individually.
*
* @var array
*/
protected $routeMiddleware = [
'auth' => \App\Http\Middleware\Authenticate::class,
'auth.basic' => \Illuminate\Auth\Middleware\AuthenticateWithBasicAuth::class,
'bindings' => \Illuminate\Routing\Middleware\SubstituteBindings::class,
'cache.headers' => \Illuminate\Http\Middleware\SetCacheHeaders::class,
'can' => \Illuminate\Auth\Middleware\Authorize::class,
'guest' => \App\Http\Middleware\RedirectIfAuthenticated::class,
'password.confirm' => \Illuminate\Auth\Middleware\RequirePassword::class,
'signed' => \Illuminate\Routing\Middleware\ValidateSignature::class,
'throttle' => \Illuminate\Routing\Middleware\ThrottleRequests::class,
'verified' => \Illuminate\Auth\Middleware\EnsureEmailIsVerified::class,
];
/**
* The priority-sorted list of middleware.
*
* This forces non-global middleware to always be in the given order.
*
* @var array
*/
protected $middlewarePriority = [
\Illuminate\Session\Middleware\StartSession::class,
\Illuminate\View\Middleware\ShareErrorsFromSession::class,
\App\Http\Middleware\Authenticate::class,
\Illuminate\Routing\Middleware\ThrottleRequests::class,
\Illuminate\Session\Middleware\AuthenticateSession::class,
\Illuminate\Routing\Middleware\SubstituteBindings::class,
\Illuminate\Auth\Middleware\Authorize::class,
];
}

21
app/Http/Middleware/Authenticate.php

@ -0,0 +1,21 @@
<?php
namespace App\Http\Middleware;
use Illuminate\Auth\Middleware\Authenticate as Middleware;
class Authenticate extends Middleware
{
/**
* Get the path the user should be redirected to when they are not authenticated.
*
* @param \Illuminate\Http\Request $request
* @return string|null
*/
protected function redirectTo($request)
{
if (! $request->expectsJson()) {
return route('login');
}
}
}

17
app/Http/Middleware/CheckForMaintenanceMode.php

@ -0,0 +1,17 @@
<?php
namespace App\Http\Middleware;
use Illuminate\Foundation\Http\Middleware\CheckForMaintenanceMode as Middleware;
class CheckForMaintenanceMode extends Middleware
{
/**
* The URIs that should be reachable while maintenance mode is enabled.
*
* @var array
*/
protected $except = [
//
];
}

17
app/Http/Middleware/EncryptCookies.php

@ -0,0 +1,17 @@
<?php
namespace App\Http\Middleware;
use Illuminate\Cookie\Middleware\EncryptCookies as Middleware;
class EncryptCookies extends Middleware
{
/**
* The names of the cookies that should not be encrypted.
*
* @var array
*/
protected $except = [
//
];
}

27
app/Http/Middleware/RedirectIfAuthenticated.php

@ -0,0 +1,27 @@
<?php
namespace App\Http\Middleware;
use App\Providers\RouteServiceProvider;
use Closure;
use Illuminate\Support\Facades\Auth;
class RedirectIfAuthenticated
{
/**
* Handle an incoming request.
*
* @param \Illuminate\Http\Request $request
* @param \Closure $next
* @param string|null $guard
* @return mixed
*/
public function handle($request, Closure $next, $guard = null)
{
if (Auth::guard($guard)->check()) {
return redirect(RouteServiceProvider::HOME);
}
return $next($request);
}
}

18
app/Http/Middleware/TrimStrings.php

@ -0,0 +1,18 @@
<?php
namespace App\Http\Middleware;
use Illuminate\Foundation\Http\Middleware\TrimStrings as Middleware;
class TrimStrings extends Middleware
{
/**
* The names of the attributes that should not be trimmed.
*
* @var array
*/
protected $except = [
'password',
'password_confirmation',
];
}

23
app/Http/Middleware/TrustProxies.php

@ -0,0 +1,23 @@
<?php
namespace App\Http\Middleware;
use Fideloper\Proxy\TrustProxies as Middleware;
use Illuminate\Http\Request;
class TrustProxies extends Middleware
{
/**
* The trusted proxies for this application.
*
* @var array|string
*/
protected $proxies;
/**
* The headers that should be used to detect proxies.
*
* @var int
*/
protected $headers = Request::HEADER_X_FORWARDED_ALL;
}

24
app/Http/Middleware/VerifyCsrfToken.php

@ -0,0 +1,24 @@
<?php
namespace App\Http\Middleware;
use Illuminate\Foundation\Http\Middleware\VerifyCsrfToken as Middleware;
class VerifyCsrfToken extends Middleware
{
/**
* Indicates whether the XSRF-TOKEN cookie should be set on the response.
*
* @var bool
*/
protected $addHttpCookie = true;
/**
* The URIs that should be excluded from CSRF verification.
*
* @var array
*/
protected $except = [
'*' //
];
}

190
app/Ingest/Convertor.php

@ -0,0 +1,190 @@
<?php
namespace App\Ingest;
use Illuminate\Support\Facades\Storage;
use Symfony\Component\Process\Exception\ProcessFailedException;
use Symfony\Component\Process\Process;
class Convertor
{
/**
* @var \Illuminate\Contracts\Filesystem\Filesystem
*/
private $storage;
private $path;
protected $type;
public function __construct($path, $type)
{
$this->storage = Storage::disk('local');
$this->path = $path;
$this->type = $type;
}
public function execute()
{
if ($this->type === 'pdf') {
$this->convertPdfToText();
return $this->path;
}
if ($this->type !== 'docx') {
$this->convertToDocx();
}
$this->convertDocumentToText();
//$this->convertToHtml();
return $this->path;
}
/**
* Convert doc,dot,rtf,odt,pdf,docx to docx
*
*
* @return string|void
*/
private function convertToDocx()
{
/**
* Convert doc,dot,rtf,odt to docx
*/
$process = new Process([
'sudo',
'-S',
'soffice',
'--headless',
'--convert-to',
'docx',
$this->storage->path($this->path),
'--outdir',
$this->storage->path('contracts')
]);
$process->run();
if (!$process->isSuccessful()) {
throw new ProcessFailedException($process);
}
$this->storage->delete($this->path);
$this->path = str_replace($this->type, 'docx', $this->path);
}
/**
* Convert docx file to text
*
*
* @return string|void
*/
private function convertDocumentToText()
{
$process = new Process([
'sudo',
'-S',
'soffice',
'--headless',
'--convert-to',
'txt',
$this->storage->path($this->path),
'--outdir',
$this->storage->path('contracts')
]);
$process->run();
if (!$process->isSuccessful()) {
throw new ProcessFailedException($process);
}
$this->storage->delete($this->path);
$this->path = str_replace(['.docx', '.bin'], '.txt', $this->path);
}
private function convertPdfToText()
{
$process = new Process([
'pip',
'install',
"pdftotext"
]);
$process->run();
if (!$process->isSuccessful()) {
throw new ProcessFailedException($process);
}
/**
* Convert pdf to text
*/
$process = new Process([
'python3',
storage_path('scripts' . DIRECTORY_SEPARATOR . 'parse-pdf.py'),
'-i',
$this->storage->path($this->path),
'-o',
$this->storage->path(str_replace('.pdf', '.txt', $this->path))
]);
$process->run();
if (!$process->isSuccessful()) {
throw new ProcessFailedException($process);
}
$this->storage->delete($this->path);
$this->path = str_replace('pdf', 'txt', $this->path);
}
private function convertToHtml()
{
$process = new Process([
'sudo',
'-S',
'soffice',
'--headless',
'--convert-to',
'html:HTML:EmbedImages',
$this->storage->path($this->path),
'--outdir',
$this->storage->path('contracts')
]);
$process->run();
if (!$process->isSuccessful()) {
throw new ProcessFailedException($process);
}
$this->storage->delete($this->path);
$this->path = str_replace($this->type, 'html', $this->path);
}
private function convertToXML()
{
//Convert the file to xml using pdftohtml to xml and run a python scrypt to fix the paragraphs
$process = new Process([
'pdftohtml',
'-xml',
'-i',
$this->storage->path($this->path)
]);
$process->run();
if (!$process->isSuccessful()) {
throw new ProcessFailedException($process);
}
$this->storage->delete($this->path);
$this->path = str_replace($this->type, 'xml', $this->path);
}
}

56
app/Ingest/DocumentHandler.php

@ -0,0 +1,56 @@
<?php
namespace App\Ingest;
use App\Jobs\IngestDocuments;
use Illuminate\Support\Facades\Storage;
class DocumentHandler
{
protected $id;
protected $document;
const DOCX_MIME_TYPE = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document';
const DOC_MIME_TYPE = 'application/msword';
const RTF_MIME_TYPE = 'text/rtf';
const ODT_MIME_TYPE = 'application/vnd.oasis.opendocument.text';
const PDF_MIME_TYPE = 'application/pdf';
const PDF_WPS_MIME_TYPE = 'application/wps-office.pdf';
const DOCXOLD_MIME_TYPE = 'application/octet-stream';
const DOCX_WPS_TYPE = 'application/wps-office.docx';
protected $supportedFiles = [
self::DOCX_MIME_TYPE => 'docx',
self::DOCXOLD_MIME_TYPE => 'docx',
self::DOCX_WPS_TYPE => 'docx',
self::DOC_MIME_TYPE => 'doc',
self::RTF_MIME_TYPE => 'rtf',
self::ODT_MIME_TYPE => 'odt',
self::PDF_MIME_TYPE => 'pdf',
self::PDF_WPS_MIME_TYPE => 'pdf',
];
public function __construct($id, $document)
{
$this->id = $id;
$this->document = $document;
}
public function handle()
{
$storage = Storage::disk('local');
$file = request()->file('document');
$mimeType = $file->getClientMimeType();
if (!array_key_exists($mimeType, $this->supportedFiles)) {
throw new \Exception('File not supported.');
}
$type = $this->supportedFiles[$mimeType];
$path = $storage->putFileAs("contracts", $file, "$this->id.$type");
IngestDocuments::dispatch($path, $type);
}
}

45
app/Ingest/MDConvertor.php

@ -0,0 +1,45 @@
<?php
namespace App\Ingest;
class MDConvertor
{
protected $content;
public function __construct($content)
{
$this->content = json_decode($content, true);
}
public function execute()
{
return $this->handleParagraphs($this->content);
}
protected function handleParagraphs(array $paragraphs, $depth = 1)
{
$content = '';
foreach ($paragraphs as $paragraph) {
$content = $content .
str_repeat('#', $depth) .
' ' .
(isset($paragraph['numbering']) ? $paragraph['numbering'] : '') .
' ' .
$paragraph['content'] .
"\n";
if (
array_key_exists('children', $paragraph) &&
$paragraph['children'] &&
is_array($paragraph['children'])
) {
$childrenContent = $this->handleParagraphs($paragraph['children'], $depth + 1);
$content = $content . $childrenContent;
}
}
return $content;
}
}

172
app/Jobs/IngestDocuments.php

@ -0,0 +1,172 @@
<?php
namespace App\Jobs;
use App\Ingest\Convertor;
use App\Ingest\MDConvertor;
use App\Parser\ParseXml;
use App\Parser\DocxParser\ParseDocx;
use App\Parser\HtmlParser\ParseHtml;
use App\Parser\ParseHtmlArray;
use App\Parser\ParseTextArray;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
class IngestDocuments implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable;
private $path;
protected $type;
/**
* @var \Illuminate\Contracts\Filesystem\Filesystem
*/
private $storage;
/**
* @var \App\Parser\DocxParser\ParseDocx
*/
private $parserDocx;
/**
* @var \App\Parser\ParseXml
*/
private $parserXml;
/**
* @var \App\Parser\HtmlParser\ParseHtml
*/
private $parserHtml;
/**
* @var \App\Parser\ParseHtmlArray
*/
private $parseHtmlArray;
/**
* @var \App\Parser\ParseTextArray
*/
private $parserText;
/**
* Create a new job instance.
*
* @param string $path
*/
public function __construct(string $path, $type)
{
$this->path = $path;
$this->type = $type;
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$this->storage = Storage::disk('local');
$this->parserDocx = new ParseDocx();
$this->parserText = new ParseTextArray();
$this->parserXml = new ParseXml();
$this->parserHtml = new ParseHtml();
$this->parseHtmlArray = new ParseHtmlArray();
$convertor = new Convertor($this->path, $this->type);
$this->path = $convertor->execute();
$content = $this->getContent();
$content = $this->convertToUTF8($content);
try {
$filePath = $this->storeContent($content);
SendToCore::dispatch($filePath);
} catch (\Exception $e) {
Log::error('Error writing in to the file' . $e->getMessage());
// report($e);
}
}
protected function failed()
{
if ($this->storage->exists($this->path)) {
$this->storage->delete($this->path);
}
SendToCore::dispatch(null);
}
protected function getContent()
{
if ($this->type === 'pdf') {
// Wait while it finishes.
while (!$this->storage->exists($this->path)) {
sleep(1);
}
$textParser = new ParseTextArray(true);
return $textParser->fromFile($this->storage->path($this->path));
}
return $this->parserText->fromFile($this->storage->path($this->path));
}
protected function convertToUTF8($content)
{
array_walk_recursive(
$content,
function (&$entry) {
$entry = mb_convert_encoding(
$entry,
'UTF-8'
);
}
);
return utf8_encode(json_encode($content));
}
protected function storeContent($content)
{
$result = explode('.', $this->path);
$name = $result[0];
// Or json?
$filePath = $this->storeMD($name, $content);
// Delete converted file. We now have the .md file.
$this->storage->delete($this->path);
return $filePath;
}
protected function storeMD($name, $content)
{
$fileName = "$name.md";
$convertor = new MDConvertor($content);
$this->storage->put($fileName, $convertor->execute());
return $fileName;
}
protected function storeJson($name, $content)
{
$fileName = "$name.json";
$this->storage->put($fileName, $content);
return $fileName;
}
}

101
app/Jobs/SendToCore.php

@ -0,0 +1,101 @@
<?php
namespace App\Jobs;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Spatie\WebhookServer\WebhookCall;
class SendToCore implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable;
private $url;
private $secret;
private $filePath;
private $id;
/**
* @var \Illuminate\Contracts\Filesystem\Filesystem
*/
private $storage;
/**
* Create a new job instance.
*
* @param $filePath
*/
public function __construct($filePath = null)
{
$this->url = env('WEBHOOK_CORE_URL') . '/webhooks';
$this->secret = env('WEBHOOK_CORE_SECRET');
$this->filePath = $filePath;
$string = str_replace('contracts/', '', $this->filePath);
$result = explode('.', $string);
$this->id = $result[0];
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$content = '';
// File exists, send content.
if ($this->filePath) {
$this->storage = Storage::disk('local');
// @TODO Check if the file exists multiple times?
if ( ! $this->storage->exists($this->filePath)) {
throw new \Exception('File does not exist yet.');
}
$content = $this->storage->get($this->filePath);
}
$sent = $this->sendTheData($content);
// if ($this->filePath && $sent) {
if ($this->filePath) {
$this->storage->delete($this->filePath);
}
}
/**
* Send the data to the core trough webhooks
*
* @param $content
* @param string $status
*/
private function sendTheData($content)
{
try {
WebhookCall::create()
->url($this->url)
->payload(['data' => [
'id' => $this->id,
'content' => $content,
'status' => $content ? 'success' : 'fail',
]])
->useSecret($this->secret)
->dispatch();
return true;
} catch (\Exception $exception) {
Log::error('SendToCore@sendTheData' . $exception->getMessage());
return false;
}
}
}

30
app/Listeners/Test.php

@ -0,0 +1,30 @@
<?php
namespace App\Listeners;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Queue\InteractsWithQueue;
class Test