Notification & Integrations refactoring (#346)

* Integrations Refactoring - WIP

* integrations list & edit - WIP

* Fix integration store binding issue

* integrations refactor - WIP

* Form integration - WIP

* Form integration Edit - WIP

* Integration Refactor - Slack - WIP

* Integration Refactor - Discord - WIP

* Integration Refactor - Webhook - WIP

* Integration Refactor - Send Submission Confirmation - WIP

* Integration Refactor - Backend handler - WIP

* Form Integration Status field

* Integration Refactor - Backend SubmissionConfirmation - WIP

* IntegrationMigration Command

* skip confirmation email test case

* Small refactoring

* FormIntegration status active/inactive

* formIntegrationData to integrationData

* Rename file name with Integration suffix for integration realted files

* Loader on form integrations

* WIP

* form integration test case

* WIP

* Added Integration card - working on refactoring

* change location for IntegrationCard and update package file

* Form Integration Create/Edit in single Modal

* Remove integration extra pages

* crisp_help_page_slug for integration json

* integration logic as collapse

* UI improvements

* WIP

* Trying to debug vue devtools

* WIP for integrations

* getIntegrationHandler change namespace name

* useForm for integration fields + validation structure

* Integration Test case & apply validation rules

* Apply useform changes to integration other files

* validation rules for FormNotificationsMessageActions fields

* Zapier integration as coming soon

* Update FormCleaner

* set default settings for confirmation integration

* WIP

* Finish validation for all integrations

* Updated purify, added integration formatData

* Fix testcase

* Ran pint; working on integration errors

* Handle integration events

* command for Delete Old Integration Events

* Display Past Events in Modal

* on Integration event create with status error send email to form creator

* Polish styling

* Minor improvements

* Finish badge and integration status

* Fix tests and add an integration event test

* Lint

---------

Co-authored-by: Julien Nahum <julien@nahum.net>

app/Service/Forms/FormCleaner.php

@@ -28,12 +28,8 @@ class FormCleaner
     private array $customKeys = ['seo_meta'];
 
     private array $formDefaults = [
-        'notifies' => false,
         'no_branding' => false,
-        'webhook_url' => null,
         'database_fields_update' => null,
-        'slack_webhook_url' => null,
-        'discord_webhook_url' => null,
         'editable_submissions' => false,
         'custom_code' => null,
         'seo_meta' => [],
@@ -46,12 +42,8 @@ class FormCleaner
 
     private array $cleaningMessages = [
         // For form
-        'notifies' => 'Email notification were disabled.',
         'no_branding' => 'OpenForm branding is not hidden.',
-        'webhook_url' => 'Webhook disabled.',
         'database_fields_update' => 'Form submission will only create new records (no updates).',
-        'slack_webhook_url' => 'Slack webhook disabled.',
-        'discord_webhook_url' => 'Discord webhook disabled.',
         'editable_submissions' => 'Users will not be able to edit their submissions.',
         'custom_code' => 'Custom code was disabled',
         'seo_meta' => 'Custom SEO was disabled',
@@ -126,7 +118,7 @@ class FormCleaner
      */
     public function simulateCleaning(Workspace $workspace): FormCleaner
     {
-        if (! $this->isPro($workspace)) {
+        if (!$this->isPro($workspace)) {
             $this->data = $this->removeProFeatures($this->data, true);
         }
 
@@ -141,7 +133,7 @@ class FormCleaner
      */
     public function performCleaning(Workspace $workspace): FormCleaner
     {
-        if (! $this->isPro($workspace)) {
+        if (!$this->isPro($workspace)) {
             $this->data = $this->removeProFeatures($this->data);
         }
 
@@ -217,14 +209,14 @@ class FormCleaner
             $formVal = (($formVal === 0 || $formVal === '0') ? false : $formVal);
             $formVal = (($formVal === 1 || $formVal === '1') ? true : $formVal);
 
-            if (! is_null($formVal) && $formVal !== $value) {
-                if (! isset($this->cleanings['form'])) {
+            if (!is_null($formVal) && $formVal !== $value) {
+                if (!isset($this->cleanings['form'])) {
                     $this->cleanings['form'] = [];
                 }
                 $this->cleanings['form'][] = $key;
 
                 // If not a simulation, do the cleaning
-                if (! $simulation) {
+                if (!$simulation) {
                     Arr::set($data, $key, $value);
                 }
             }
@@ -236,7 +228,7 @@ class FormCleaner
         foreach ($defaults as $key => $value) {
             if (isset($data[$key]) && Arr::get($data, $key) !== $value) {
                 $this->cleanings[$data['name']][] = $key;
-                if (! $simulation) {
+                if (!$simulation) {
                     Arr::set($data, $key, $value);
                 }
             }

app/Service/Forms/Integrations/AbstractIntegrationHandler.php

@@ -0,0 +1,131 @@
+<?php
+
+namespace App\Service\Forms\Integrations;
+
+use App\Models\Integration\FormIntegration;
+use App\Events\Forms\FormSubmitted;
+use App\Models\Integration\FormIntegrationsEvent;
+use App\Service\Forms\FormSubmissionFormatter;
+use App\Service\Forms\FormLogicConditionChecker;
+use Illuminate\Http\Client\RequestException;
+use Illuminate\Support\Facades\Http;
+use Vinkla\Hashids\Facades\Hashids;
+
+abstract class AbstractIntegrationHandler
+{
+    protected $form = null;
+    protected $submissionData = null;
+    protected $integrationData = null;
+
+    public function __construct(
+        protected FormSubmitted $event,
+        protected FormIntegration $formIntegration,
+        protected array $integration
+    ) {
+        $this->form = $event->form;
+        $this->submissionData = $event->data;
+        $this->integrationData = $formIntegration->data;
+    }
+
+    protected function getProviderName(): string
+    {
+        return $this->integration['name'] ?? '';
+    }
+
+    protected function logicConditionsMet(): bool
+    {
+        if (!$this->formIntegration->logic) {
+            return true;
+        }
+        return FormLogicConditionChecker::conditionsMet(
+            json_decode(json_encode($this->formIntegration->logic), true),
+            $this->submissionData
+        );
+    }
+
+    protected function shouldRun(): bool
+    {
+        return $this->logicConditionsMet();
+    }
+
+    protected function getWebhookUrl(): ?string
+    {
+        return '';
+    }
+
+    /**
+     * Default webhook payload. Can be changed in child classes.
+     */
+    protected function getWebhookData(): array
+    {
+        $formatter = (new FormSubmissionFormatter($this->form, $this->submissionData))
+            ->useSignedUrlForFiles()
+            ->showHiddenFields();
+
+        $formattedData = [];
+        foreach ($formatter->getFieldsWithValue() as $field) {
+            $formattedData[$field['name']] = $field['value'];
+        }
+
+        $data = [
+            'form_title' => $this->form->title,
+            'form_slug' => $this->form->slug,
+            'submission' => $formattedData,
+        ];
+        if ($this->form->is_pro && $this->form->editable_submissions) {
+            $data['edit_link'] = $this->form->share_url . '?submission_id=' . Hashids::encode(
+                $this->submissionData['submission_id']
+            );
+        }
+
+        return $data;
+    }
+
+    final public function run(): void
+    {
+        try {
+            $this->handle();
+            $this->formIntegration->events()->create([
+                'status' => FormIntegrationsEvent::STATUS_SUCCESS,
+            ]);
+        } catch (\Exception $e) {
+            $this->formIntegration->events()->create([
+                'status' => FormIntegrationsEvent::STATUS_ERROR,
+                'data' => $this->extractEventDataFromException($e),
+            ]);
+        }
+    }
+
+    /**
+     * Default handle. Can be changed in child classes.
+     */
+    public function handle(): void
+    {
+        if (!$this->shouldRun()) {
+            return;
+        }
+
+        Http::throw()->post($this->getWebhookUrl(), $this->getWebhookData());
+    }
+
+    abstract public static function getValidationRules(): array;
+
+    public static function formatData(array $data): array
+    {
+        return $data;
+    }
+
+    public function extractEventDataFromException(\Exception $e): array
+    {
+        if ($e instanceof RequestException) {
+            return [
+                'message' => $e->getMessage(),
+                'response' => $e->response->json(),
+                'status' => $e->response->status(),
+            ];
+        }
+        return [
+            'message' => $e->getMessage()
+        ];
+    }
+}

app/Service/Forms/Integrations/AbstractWebhookHandler.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace App\Service\Forms\Webhooks;
+namespace App\Service\Forms\Integrations;
 
 use App\Models\Forms\Form;
 use App\Service\Forms\FormSubmissionFormatter;

app/Service/Forms/Integrations/DiscordHandler.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace App\Service\Forms\Webhooks;
+namespace App\Service\Forms\Integrations;
 
 use App\Service\Forms\FormSubmissionFormatter;
 use Illuminate\Support\Arr;

app/Service/Forms/Integrations/DiscordIntegration.php

@@ -0,0 +1,91 @@
+<?php
+
+namespace App\Service\Forms\Integrations;
+
+use App\Service\Forms\FormSubmissionFormatter;
+use Illuminate\Support\Arr;
+use Vinkla\Hashids\Facades\Hashids;
+
+class DiscordIntegration extends AbstractIntegrationHandler
+{
+    public static function getValidationRules(): array
+    {
+        return [
+            'discord_webhook_url' => 'required|url|starts_with:https://discord.com/api/webhooks',
+            'include_submission_data' => 'boolean',
+            'link_open_form' => 'boolean',
+            'link_edit_form' => 'boolean',
+            'views_submissions_count' => 'boolean',
+            'link_edit_submission' => 'boolean'
+        ];
+    }
+
+    protected function getWebhookUrl(): ?string
+    {
+        return $this->integrationData->discord_webhook_url;
+    }
+
+    protected function shouldRun(): bool
+    {
+        return !is_null($this->getWebhookUrl()) && $this->form->is_pro && parent::shouldRun();
+    }
+
+    protected function getWebhookData(): array
+    {
+        $settings = (array) $this->integrationData ?? [];
+        $externalLinks = [];
+        if (Arr::get($settings, 'link_open_form', true)) {
+            $externalLinks[] = '[**🔗 Open Form**](' . $this->form->share_url . ')';
+        }
+        if (Arr::get($settings, 'link_edit_form', true)) {
+            $editFormURL = front_url('forms/' . $this->form->slug . '/show');
+            $externalLinks[] = '[**✍️ Edit Form**](' . $editFormURL . ')';
+        }
+        if (Arr::get($settings, 'link_edit_submission', true) && $this->form->editable_submissions) {
+            $submissionId = Hashids::encode($this->submissionData['submission_id']);
+            $externalLinks[] = '[**✍️ ' . $this->form->editable_submissions_button_text . '**](' . $this->form->share_url . '?submission_id=' . $submissionId . ')';
+        }
+
+        $color = hexdec(str_replace('#', '', $this->form->color));
+        $blocks = [];
+        if (Arr::get($settings, 'include_submission_data', true)) {
+            $submissionString = '';
+            $formatter = (new FormSubmissionFormatter($this->form, $this->submissionData))->outputStringsOnly();
+            foreach ($formatter->getFieldsWithValue() as $field) {
+                $tmpVal = is_array($field['value']) ? implode(',', $field['value']) : $field['value'];
+                $submissionString .= '**' . ucfirst($field['name']) . '**: ' . $tmpVal . "\n";
+            }
+            $blocks[] = [
+                'type' => 'rich',
+                'color' => $color,
+                'description' => $submissionString,
+            ];
+        }
+
+        if (Arr::get($settings, 'views_submissions_count', true)) {
+            $countString = '**👀 Views**: ' . (string) $this->form->views_count . " \n";
+            $countString .= '**🖊️ Submissions**: ' . (string) $this->form->submissions_count;
+            $blocks[] = [
+                'type' => 'rich',
+                'color' => $color,
+                'description' => $countString,
+            ];
+        }
+
+        if (count($externalLinks) > 0) {
+            $blocks[] = [
+                'type' => 'rich',
+                'color' => $color,
+                'description' => implode(' - ', $externalLinks),
+            ];
+        }
+
+        return [
+            'content' => 'New submission for your form **' . $this->form->title . '**',
+            'tts' => false,
+            'username' => config('app.name'),
+            'avatar_url' => asset('img/logo.png'),
+            'embeds' => $blocks,
+        ];
+    }
+}

app/Service/Forms/Integrations/EmailIntegration.php

@@ -0,0 +1,46 @@
+<?php
+
+namespace App\Service\Forms\Integrations;
+
+use App\Rules\OneEmailPerLine;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Notification;
+use App\Notifications\Forms\FormSubmissionNotification;
+
+class EmailIntegration extends AbstractIntegrationHandler
+{
+    public static function getValidationRules(): array
+    {
+        return [
+            'notification_emails' => ['required', new OneEmailPerLine()],
+            'notification_reply_to' => 'email|nullable',
+        ];
+    }
+
+    protected function shouldRun(): bool
+    {
+        return !(!$this->form->is_pro || !$this->integrationData->notification_emails) && parent::shouldRun();
+    }
+
+    public function handle(): void
+    {
+        if (!$this->shouldRun()) {
+            return;
+        }
+
+        $subscribers = collect(preg_split("/\r\n|\n|\r/", $this->integrationData->notification_emails))
+            ->filter(function ($email) {
+                return filter_var($email, FILTER_VALIDATE_EMAIL);
+            });
+        Log::debug('Sending email notification', [
+            'recipients' => $subscribers->toArray(),
+            'form_id' => $this->form->id,
+            'form_slug' => $this->form->slug,
+        ]);
+        $subscribers->each(function ($subscriber) {
+            Notification::route('mail', $subscriber)->notify(
+                new FormSubmissionNotification($this->event, $this->integrationData)
+            );
+        });
+    }
+}

app/Service/Forms/Integrations/SimpleWebhookHandler.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace App\Service\Forms\Webhooks;
+namespace App\Service\Forms\Integrations;
 
 class SimpleWebhookHandler extends AbstractWebhookHandler
 {

app/Service/Forms/Integrations/SlackHandler.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace App\Service\Forms\Webhooks;
+namespace App\Service\Forms\Integrations;
 
 use App\Service\Forms\FormSubmissionFormatter;
 use Illuminate\Support\Arr;

app/Service/Forms/Integrations/SlackIntegration.php

@@ -0,0 +1,101 @@
+<?php
+
+namespace App\Service\Forms\Integrations;
+
+use App\Service\Forms\FormSubmissionFormatter;
+use Illuminate\Support\Arr;
+use Vinkla\Hashids\Facades\Hashids;
+
+class SlackIntegration extends AbstractIntegrationHandler
+{
+    public static function getValidationRules(): array
+    {
+        return [
+            'slack_webhook_url' => 'required|url|starts_with:https://hooks.slack.com/',
+            'include_submission_data' => 'boolean',
+            'link_open_form' => 'boolean',
+            'link_edit_form' => 'boolean',
+            'views_submissions_count' => 'boolean',
+            'link_edit_submission' => 'boolean'
+        ];
+    }
+
+    protected function getWebhookUrl(): ?string
+    {
+        return $this->integrationData->slack_webhook_url;
+    }
+
+    protected function shouldRun(): bool
+    {
+        return !is_null($this->getWebhookUrl()) && $this->form->is_pro && parent::shouldRun();
+    }
+
+    protected function getWebhookData(): array
+    {
+        $settings = (array) $this->integrationData ?? [];
+        $externalLinks = [];
+        if (Arr::get($settings, 'link_open_form', true)) {
+            $externalLinks[] = '*<' . $this->form->share_url . '|🔗 Open Form>*';
+        }
+        if (Arr::get($settings, 'link_edit_form', true)) {
+            $editFormURL = front_url('forms/' . $this->form->slug . '/show');
+            $externalLinks[] = '*<' . $editFormURL . '|✍️ Edit Form>*';
+        }
+        if (Arr::get($settings, 'link_edit_submission', true) && $this->form->editable_submissions) {
+            $submissionId = Hashids::encode($this->submissionData['submission_id']);
+            $externalLinks[] = '*<' . $this->form->share_url . '?submission_id=' . $submissionId . '|✍️ ' . $this->form->editable_submissions_button_text . '>*';
+        }
+
+        $blocks = [
+            [
+                'type' => 'section',
+                'text' => [
+                    'type' => 'mrkdwn',
+                    'text' => 'New submission for your form *' . $this->form->title . '*',
+                ],
+            ],
+        ];
+
+        if (Arr::get($settings, 'include_submission_data', true)) {
+            $submissionString = '';
+            $formatter = (new FormSubmissionFormatter($this->form, $this->submissionData))->outputStringsOnly();
+            foreach ($formatter->getFieldsWithValue() as $field) {
+                $tmpVal = is_array($field['value']) ? implode(',', $field['value']) : $field['value'];
+                $submissionString .= '>*' . ucfirst($field['name']) . '*: ' . $tmpVal . " \n";
+            }
+            $blocks[] = [
+                'type' => 'section',
+                'text' => [
+                    'type' => 'mrkdwn',
+                    'text' => $submissionString,
+                ],
+            ];
+        }
+
+        if (Arr::get($settings, 'views_submissions_count', true)) {
+            $countString = '*👀 Views*: ' . (string) $this->form->views_count . " \n";
+            $countString .= '*🖊️ Submissions*: ' . (string) $this->form->submissions_count;
+            $blocks[] = [
+                'type' => 'section',
+                'text' => [
+                    'type' => 'mrkdwn',
+                    'text' => $countString,
+                ],
+            ];
+        }
+
+        if (count($externalLinks) > 0) {
+            $blocks[] = [
+                'type' => 'section',
+                'text' => [
+                    'type' => 'mrkdwn',
+                    'text' => implode('     ', $externalLinks),
+                ],
+            ];
+        }
+
+        return [
+            'blocks' => $blocks,
+        ];
+    }
+}

app/Service/Forms/Integrations/SubmissionConfirmationIntegration.php

@@ -0,0 +1,103 @@
+<?php
+
+namespace App\Service\Forms\Integrations;
+
+use App\Mail\Forms\SubmissionConfirmationMail;
+use Illuminate\Support\Facades\Mail;
+use Illuminate\Support\Facades\Log;
+use Stevebauman\Purify\Facades\Purify;
+
+/**
+ * Sends a confirmation to form respondant that form was submitted
+ */
+class SubmissionConfirmationIntegration extends AbstractIntegrationHandler
+{
+    public const RISKY_USERS_LIMIT = 120;
+
+    public static function getValidationRules(): array
+    {
+        return [
+            'confirmation_reply_to' => 'email|nullable',
+            'notification_sender' => 'required',
+            'notification_subject' => 'required',
+            'notification_body' => 'required',
+            'notifications_include_submission' => 'boolean'
+        ];
+    }
+
+    protected function shouldRun(): bool
+    {
+        return !(!$this->form->is_pro) && parent::shouldRun() && !$this->riskLimitReached();
+    }
+
+    public function handle(): void
+    {
+        if (!$this->shouldRun()) {
+            return;
+        }
+
+        $email = $this->getRespondentEmail();
+        if (!$email) {
+            return;
+        }
+
+        Log::info('Sending submission confirmation', [
+            'recipient' => $email,
+            'form_id' => $this->form->id,
+            'form_slug' => $this->form->slug,
+        ]);
+        Mail::to($email)->send(new SubmissionConfirmationMail($this->event, $this->integrationData));
+    }
+
+    private function getRespondentEmail()
+    {
+        // Make sure we only have one email field in the form
+        $emailFields = collect($this->form->properties)->filter(function ($field) {
+            $hidden = $field['hidden'] ?? false;
+
+            return !$hidden && $field['type'] == 'email';
+        });
+        if ($emailFields->count() != 1) {
+            return null;
+        }
+
+        if (isset($this->submissionData[$emailFields->first()['id']])) {
+            $email = $this->submissionData[$emailFields->first()['id']];
+            if ($this->validateEmail($email)) {
+                return $email;
+            }
+        }
+
+        return null;
+    }
+
+    // To avoid phishing abuse we limit this feature for risky users
+    private function riskLimitReached(): bool
+    {
+        // This is a per-workspace limit for risky workspaces
+        if ($this->form->workspace->is_risky) {
+            if ($this->form->workspace->submissions_count >= self::RISKY_USERS_LIMIT) {
+                Log::error('!!!DANGER!!! Dangerous user detected! Attempting many email sending.', [
+                    'form_id' => $this->form->id,
+                    'workspace_id' => $this->form->workspace->id,
+                ]);
+
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    public static function validateEmail($email): bool
+    {
+        return (bool)filter_var($email, FILTER_VALIDATE_EMAIL);
+    }
+
+    public static function formatData(array $data): array
+    {
+        return array_merge(parent::formatData($data), [
+            'notification_body' => Purify::clean($data['notification_body'] ?? ''),
+        ]);
+    }
+}

app/Service/Forms/Integrations/WebhookHandlerProvider.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace App\Service\Forms\Webhooks;
+namespace App\Service\Forms\Integrations;
 
 use App\Models\Forms\Form;
 

app/Service/Forms/Integrations/WebhookIntegration.php

@@ -0,0 +1,23 @@
+<?php
+
+namespace App\Service\Forms\Integrations;
+
+class WebhookIntegration extends AbstractIntegrationHandler
+{
+    public static function getValidationRules(): array
+    {
+        return [
+            'webhook_url' => 'required|url'
+        ];
+    }
+
+    protected function getWebhookUrl(): ?string
+    {
+        return $this->integrationData->webhook_url;
+    }
+
+    protected function shouldRun(): bool
+    {
+        return !is_null($this->getWebhookUrl()) && parent::shouldRun();
+    }
+}

app/Service/Forms/Integrations/ZapierHandler.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace App\Service\Forms\Webhooks;
+namespace App\Service\Forms\Integrations;
 
 use App\Models\Forms\Form;