Better Form Stats (#567)

* Better Form Stats

* fix lint

* submission timer store in localstorage

* Update test case for stats

* remove extra code

* fix form stats

* on restart remove timer

* fix resetTimer function name

* Improve form timer

* Fix timer after form validation error + polish UI

---------

Co-authored-by: Julien Nahum <julien@nahum.net>

api/app/Http/Controllers/Forms/FormStatsController.php

@@ -3,7 +3,9 @@
 namespace App\Http\Controllers\Forms;
 
 use App\Http\Controllers\Controller;
+use App\Http\Requests\FormStatsRequest;
 use Carbon\CarbonPeriod;
+use Carbon\CarbonInterval;
 use Illuminate\Http\Request;
 
 class FormStatsController extends Controller
@@ -13,14 +15,14 @@ class FormStatsController extends Controller
         $this->middleware('auth');
     }
 
-    public function getFormStats(Request $request)
+    public function getFormStats(FormStatsRequest $request)
     {
         $form = $request->form; // Added by ProForm middleware
         $this->authorize('view', $form);
 
-        $formStats = $form->statistics()->where('date', '>', now()->subDays(29)->startOfDay())->get();
+        $formStats = $form->statistics()->whereBetween('date', [$request->date_from, $request->date_to])->get();
         $periodStats = ['views' => [], 'submissions' => []];
-        foreach (CarbonPeriod::create(now()->subDays(29), now()) as $dateObj) {
+        foreach (CarbonPeriod::create($request->date_from, $request->date_to) as $dateObj) {
             $date = $dateObj->format('d-m-Y');
 
             $statisticData = $formStats->where('date', $dateObj->format('Y-m-d'))->first();
@@ -34,4 +36,26 @@ class FormStatsController extends Controller
 
         return $periodStats;
     }
+
+    public function getFormStatsDetails(Request $request)
+    {
+        $form = $request->form; // Added by ProForm middleware
+        $this->authorize('view', $form);
+
+        $totalViews = $form->views()->count();
+        $totalSubmissions = $form->submissions_count;
+
+        $averageDuration = \Cache::remember('form_stats_average_duration_' . $form->id, 1800, function () use ($form) {
+            $submissionsWithDuration = $form->submissions()->whereNotNull('completion_time')->count() ?? 0;
+            $totalDuration = $form->submissions()->whereNotNull('completion_time')->sum('completion_time') ?? 0;
+            return $submissionsWithDuration > 0 ? round($totalDuration / $submissionsWithDuration) : null;
+        });
+
+        return [
+            'views' => $totalViews,
+            'submissions' => $totalSubmissions,
+            'completion_rate' => $totalViews > 0 ? round(($totalSubmissions / $totalViews) * 100, 2) : 0,
+            'average_duration' => $averageDuration ? CarbonInterval::seconds($averageDuration)->cascade()->forHumans() : null
+        ];
+    }
 }

api/app/Http/Controllers/Forms/PublicFormController.php

@@ -77,7 +77,7 @@ class PublicFormController extends Controller
 
         $internal_url = Storage::temporaryUrl($path, now()->addMinutes(5));
 
-        foreach(config('filesystems.disks.s3.temporary_url_rewrites') as $from => $to) {
+        foreach (config('filesystems.disks.s3.temporary_url_rewrites') as $from => $to) {
             $internal_url = str_replace($from, $to, $internal_url);
         }
 
@@ -89,12 +89,16 @@ class PublicFormController extends Controller
         $form = $request->form;
         $submissionId = false;
 
+        $submissionData = $request->validated();
+        $completionTime = $request->get('completion_time') ?? null;
+        unset($submissionData['completion_time']); // Remove completion_time from the main data array
+
         if ($form->editable_submissions) {
-            $job = new StoreFormSubmissionJob($form, $request->validated());
+            $job = new StoreFormSubmissionJob($form, $submissionData, $completionTime);
             $job->handle();
             $submissionId = Hashids::encode($job->getSubmissionId());
         } else {
-            StoreFormSubmissionJob::dispatch($form, $request->validated());
+            StoreFormSubmissionJob::dispatch($form, $submissionData, $completionTime);
         }
 
         return $this->success(array_merge([

api/app/Http/Requests/FormStatsRequest.php

@@ -0,0 +1,31 @@
+<?php
+
+namespace App\Http\Requests;
+
+use Illuminate\Foundation\Http\FormRequest;
+use Carbon\Carbon;
+
+class FormStatsRequest extends FormRequest
+{
+    public function authorize()
+    {
+        return true;
+    }
+
+    public function rules()
+    {
+        return [
+            'date_from' => 'required|date',
+            'date_to' => 'required|date|after_or_equal:date_from',
+        ];
+    }
+
+    public function withValidator($validator)
+    {
+        $validator->after(function ($validator) {
+            if (Carbon::parse($this->date_from)->diffInMonths(Carbon::parse($this->date_to)) > 3) {
+                $validator->errors()->add('date_range', 'Date range exceeds 3 months. Please select a shorter period.');
+            }
+        });
+    }
+}

api/app/Http/Resources/FormSubmissionResource.php

@@ -23,7 +23,8 @@ class FormSubmissionResource extends JsonResource
         }
 
         return array_merge([
-            'data' => $this->data
+            'data' => $this->data,
+            'completion_time' => $this->completion_time,
         ], ($this->publiclyAccessed) ? [] : [
             'form_id' => $this->form_id,
             'id' => $this->id

api/app/Jobs/Form/StoreFormSubmissionJob.php

@@ -33,7 +33,7 @@ class StoreFormSubmissionJob implements ShouldQueue
      *
      * @return void
      */
-    public function __construct(public Form $form, public array $submissionData)
+    public function __construct(public Form $form, public array $submissionData, public ?int $completionTime = null)
     {
     }
 
@@ -70,11 +70,13 @@ class StoreFormSubmissionJob implements ShouldQueue
         // Create or update record
         if ($previousSubmission = $this->submissionToUpdate()) {
             $previousSubmission->data = $formData;
+            $previousSubmission->completion_time = $this->completionTime;
             $previousSubmission->save();
             $this->submissionId = $previousSubmission->id;
         } else {
             $response = $this->form->submissions()->create([
                 'data' => $formData,
+                'completion_time' => $this->completionTime,
             ]);
             $this->submissionId = $response->id;
         }

api/app/Models/Forms/FormSubmission.php

@@ -11,12 +11,14 @@ class FormSubmission extends Model
 
     protected $fillable = [
         'data',
+        'completion_time',
     ];
 
     protected function casts(): array
     {
         return [
             'data' => 'array',
+            'completion_time' => 'integer',
         ];
     }
 

api/database/migrations/2024_09_09_180618_add_completion_time_to_form_submissions.php

@@ -0,0 +1,27 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class () extends Migration {
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('form_submissions', function (Blueprint $table) {
+            $table->integer('completion_time')->nullable();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('form_submissions', function (Blueprint $table) {
+            $table->dropColumn('completion_time');
+        });
+    }
+};

api/routes/api.php

@@ -146,6 +146,7 @@ Route::group(['middleware' => 'auth:api'], function () {
 
                 Route::middleware('pro-form')->group(function () {
                     Route::get('form-stats/{formId}', [FormStatsController::class, 'getFormStats'])->name('form.stats');
+                    Route::get('form-stats-details/{formId}', [FormStatsController::class, 'getFormStatsDetails'])->name('form.stats-details');
                 });
             });
         });

api/tests/Feature/Forms/FormStatTest.php

@@ -39,7 +39,7 @@ it('check formstat chart data', function () {
     }
 
     // Now check chart data
-    $this->getJson(route('open.workspaces.form.stats', [$workspace->id, $form->id]))
+    $this->getJson(route('open.workspaces.form.stats', [$workspace->id, $form->id]) . '?date_from=' . now()->subDays(29)->format('Y-m-d') . '&date_to=' . now()->format('Y-m-d'))
         ->assertSuccessful()
         ->assertJson(function (\Illuminate\Testing\Fluent\AssertableJson $json) use ($views, $submissions) {
             return $json->whereType('views', 'array')
@@ -65,3 +65,37 @@ it('check formstat chart data', function () {
                 ->etc();
         });
 });
+
+
+it('checks form stats details', function () {
+    $user = $this->actingAsProUser();
+    $workspace = $this->createUserWorkspace($user);
+    $form = $this->createForm($user, $workspace, []);
+
+    // Create form submissions with varying completion times
+    $form->submissions()->createMany([
+        ['completion_time' => 60],  // 1 minute
+        ['completion_time' => 60],  // 1 minute
+        ['completion_time' => 60],  // 1 minute
+        ['completion_time' => 120], // 2 minutes
+        ['completion_time' => 120], // 2 minutes
+        [] // Incomplete submission
+    ]);
+
+    // Create form views
+    $form->views()->createMany(array_fill(0, 10, []));
+
+    $this->getJson(route('open.workspaces.form.stats-details', [$workspace->id, $form->id]))
+        ->assertSuccessful()
+        ->assertJson(function (\Illuminate\Testing\Fluent\AssertableJson $json) {
+            return $json->has('views')
+                ->has('submissions')
+                ->has('completion_rate')
+                ->has('average_duration')
+                ->where('views', 10)
+                ->where('submissions', 6)
+                ->where('completion_rate', 60)
+                ->where('average_duration', '1 minute 24 seconds')
+                ->etc();
+        });
+});

client/components/open/forms/FormTimer.vue

@@ -0,0 +1,57 @@
+<script setup>
+import { ref, onMounted, onUnmounted } from 'vue'
+import { pendingSubmission as pendingSubmissionFun } from "~/composables/forms/pendingSubmission.js"
+
+const props = defineProps({
+  form: { type: Object, required: true }
+})
+
+const pendingSubmission = pendingSubmissionFun(props.form)
+const startTime = ref(null)
+const completionTime = ref(parseInt(pendingSubmission.getTimer() ?? null))
+let timer = null
+
+watch(() => completionTime.value, () => {
+  if (completionTime.value) {
+    pendingSubmission.setTimer(completionTime.value)
+  }
+}, { immediate: true })
+
+const startTimer = () => {
+  if (!startTime.value) {
+    startTime.value = parseInt(pendingSubmission.getTimer() ?? 1)
+    completionTime.value = startTime.value
+    timer = setInterval(() => {
+      completionTime.value += 1
+    }, 1000)
+  }
+}
+
+const stopTimer = () => {
+  if (timer) {
+    clearInterval(timer)
+    timer = null
+    startTime.value = null
+  }
+}
+
+const resetTimer = () => {
+  stopTimer()
+  completionTime.value = null
+}
+
+onMounted(() => {
+  document.addEventListener('input', startTimer)
+})
+
+onUnmounted(() => {
+  document.removeEventListener('input', startTimer)
+  stopTimer()
+})
+
+defineExpose({
+  completionTime,
+  stopTimer,
+  resetTimer
+})
+</script>

client/components/open/forms/OpenCompleteForm.vue

@@ -4,6 +4,11 @@
     class="open-complete-form"
     :style="{ '--font-family': form.font_family }"
   >
+    <FormTimer
+      ref="formTimer"
+      :form="form"
+    />
+
     <link
       v-if="adminPreview && form.font_family"
       rel="stylesheet"
@@ -200,6 +205,7 @@
 <script>
 import OpenForm from './OpenForm.vue'
 import OpenFormButton from './OpenFormButton.vue'
+import FormTimer from './FormTimer.vue'
 import VButton from '~/components/global/VButton.vue'
 import FormCleanings from '../../pages/forms/show/FormCleanings.vue'
 import VTransition from '~/components/global/transitions/VTransition.vue'
@@ -208,7 +214,7 @@ import clonedeep from "clone-deep"
 import ThemeBuilder from "~/lib/forms/themes/ThemeBuilder.js"
 
 export default {
-  components: { VTransition, VButton, OpenFormButton, OpenForm, FormCleanings },
+  components: { VTransition, VButton, OpenFormButton, OpenForm, FormCleanings, FormTimer },
 
   props: {
     form: { type: Object, required: true },
@@ -274,8 +280,15 @@ export default {
 
       if (form.busy) return
       this.loading = true
+
+      // Stop the timer and get the completion time
+      this.$refs.formTimer.stopTimer()
+      const completionTime = this.$refs.formTimer.completionTime
+      form.completion_time = completionTime
+
       // this.closeAlert()
-      form.post('/forms/' + this.form.slug + '/answer').then((data) => {
+      form.post('/forms/' + this.form.slug + '/answer')
+      .then((data) => {
         useAmplitude().logEvent('form_submission', {
           workspace_id: this.form.workspace_id,
           form_id: this.form.id
@@ -288,7 +301,8 @@ export default {
             id: this.form.id,
             redirect_target_url: (this.form.is_pro && data.redirect && data.redirect_url) ? data.redirect_url : null
           },
-          submission_data: form.data()
+          submission_data: form.data(),
+          completion_time: completionTime
         })
 
         if (this.isIframe) {
@@ -296,6 +310,7 @@ export default {
         }
         window.postMessage(payload, '*')
         this.pendingSubmission.remove()
+        this.pendingSubmission.removeTimer()
 
         if (data.redirect && data.redirect_url) {
           window.location.href = data.redirect_url
@@ -319,12 +334,14 @@ export default {
           useAlert().error(error.data.message)
         }
         this.loading = false
+        this.$refs.formTimer.startTimer()
         onFailure()
       })
     },
     restart () {
       this.submitted = false
       this.$emit('restarted', true)
+      this.$refs.formTimer.resetTimer() // Reset the timer
     },
     passwordEntered () {
       if (this.passwordForm.password !== '' && this.passwordForm.password !== null) {

client/components/open/forms/components/FormStats.vue

@@ -1,45 +1,60 @@
 <template>
-  <div
+  <div>
-    class="border border-nt-blue-light bg-blue-50 dark:bg-notion-dark-light rounded-md p-4 mb-5 w-full mx-auto mt-4 select-all"
+    <div class="flex flex-wrap items-end mt-5">
-  >
+      <h3 class="flex-grow font-medium text-lg mb-3">
-    <div
+        Views & Submission History
-      v-if="!form.is_pro"
+      </h3>
-      class="relative"
+      <DateInput
-    >
+        :form="filterForm"
-      <div class="absolute inset-0 z-10">
+        name="filter_date"
-        <div class="p-5 max-w-md mx-auto mt-5">
+        class="flex-1 !mb-0"
-          <p class="text-center">
+        :date-range="true"
-            You need a <pro-tag
+        :disable-future-dates="true"
-              upgrade-modal-title="Upgrade today to access form analytics"
+        :disabled="!form.is_pro"
-              class="mx-1"
+      />
-            /> subscription to access your form
+    </div>
-            analytics.
+    <div
-          </p>
+      class="border border-gray-300 rounded-lg shadow-sm p-4 mb-5 w-full mx-auto mt-4 select-all"
-          <p class="mt-5 text-center">
+    >
-            <v-button
+      <div
-              class="w-full"
+        v-if="!form.is_pro"
-              @click.prevent="subscriptionModalStore.openModal()"
+        class="relative"
-            >
+      >
-              Subscribe
+        <div class="absolute inset-0 z-10">
-            </v-button>
+          <div class="p-5 max-w-md mx-auto mt-5">
-          </p>
+            <p class="text-center">
-        </div>
+              You need a <pro-tag
-      </div>
+                upgrade-modal-title="Upgrade today to access form analytics"
-      <img
+                class="mx-1"
-        src="/img/pages/forms/blurred_graph.png"
+              /> subscription to access your form
-        alt="Sample Graph"
+              analytics.
-        class="mx-auto filter blur-md z-0"
+            </p>
-      >
+            <p class="mt-5 text-center">
+              <v-button
+                class="w-full"
+                @click.prevent="subscriptionModalStore.openModal()"
+              >
+                Subscribe
+              </v-button>
+            </p>
+          </div>
+        </div>
+        <img
+          src="/img/pages/forms/blurred_graph.png"
+          alt="Sample Graph"
+          class="mx-auto filter blur-md z-0"
+        >
+      </div>
+      <Loader
+        v-else-if="isLoading"
+        class="h-6 w-6 text-nt-blue mx-auto"
+      />
+      <LineChart
+        v-else
+        :options="chartOptions"
+        :data="chartData"
+      />
     </div>
-    <Loader
-      v-else-if="isLoading"
-      class="h-6 w-6 text-nt-blue mx-auto"
-    />
-    <LineChart
-      v-else
-      :options="chartOptions"
-      :data="chartData"
-    />
   </div>
 </template>
 
@@ -55,7 +70,6 @@ import {
   CategoryScale,
   PointElement,
 } from "chart.js"
-import ProTag from "~/components/global/ProTag.vue"
 
 ChartJS.register(
   Title,
@@ -69,10 +83,7 @@ ChartJS.register(
 
 export default {
   name: "FormStats",
-  components: {
+  components: { LineChart },
-    ProTag,
-    LineChart,
-  },
   props: {
     form: {
       type: Object,
@@ -81,8 +92,12 @@ export default {
   },
   setup() {
     const subscriptionModalStore = useSubscriptionModalStore()
+    const filterForm = useForm({
+      filter_date: null,
+    })
     return {
-      subscriptionModalStore
+      subscriptionModalStore,
+      filterForm
     }
   },
   data() {
@@ -119,8 +134,23 @@ export default {
       },
     }
   },
+  watch: {
+    filterForm: {
+      deep: true,
+      handler(newVal) {
+        if(newVal.filter_date && Array.isArray(newVal.filter_date) && newVal.filter_date[0] && newVal.filter_date[1]) {
+          this.getChartData()
+        }
+      }
+    }
+  },
   mounted() {
-    this.getChartData()
+    if (this.form.is_pro) {
+      const toDate = new Date()
+      const fromDate = new Date(toDate)
+      fromDate.setDate(toDate.getDate() - 29)
+      this.filterForm.filter_date = [fromDate.toISOString().split('T')[0], toDate.toISOString().split('T')[0]]
+    }
   },
   methods: {
     getChartData() {
@@ -131,6 +161,12 @@ export default {
           this.form.workspace_id +
           "/form-stats/" +
           this.form.id,
+        {
+          params: {
+            date_from: this.filterForm.filter_date[0] ? this.filterForm.filter_date[0].split('T')[0] : null,
+            date_to: this.filterForm.filter_date[1] ? this.filterForm.filter_date[1].split('T')[0] : null,
+          }
+        }
       ).then((statsData) => {
         if (statsData && statsData.views !== undefined) {
           this.chartData.labels = Object.keys(statsData.views)
@@ -138,6 +174,9 @@ export default {
           this.chartData.datasets[1].data = statsData.submissions
           this.isLoading = false
         }
+      }).catch((error) => {
+        this.isLoading = false
+        useAlert().error(error.data.message)
       })
     },
   },

client/composables/forms/pendingSubmission.js

@@ -7,6 +7,9 @@ export const pendingSubmission = (form) => {
       ? form.form_pending_submission_key + "-" + hash(window.location.href)
       : ""
   })
+  const formPendingSubmissionTimerKey = computed(() => {
+    return formPendingSubmissionKey.value + "-timer"
+  })
 
   const enabled = computed(() => {
     return form?.auto_save ?? false
@@ -28,10 +31,27 @@ export const pendingSubmission = (form) => {
     return pendingSubmission ? JSON.parse(pendingSubmission) : defaultValue
   }
 
+  const setTimer = (value) => {
+    if (import.meta.server) return
+    useStorage(formPendingSubmissionTimerKey.value).value = value
+  }
+
+  const removeTimer = () => {
+    return setTimer(null)
+  }
+
+  const getTimer = (defaultValue = null) => {
+    if (import.meta.server) return
+    return useStorage(formPendingSubmissionTimerKey.value).value ?? defaultValue
+  }
+
   return {
     enabled,
     set,
     get,
     remove,
+    setTimer,
+    removeTimer,
+    getTimer,
   }
 }

client/pages/forms/[slug]/show/stats.vue

@@ -1,8 +1,39 @@
 <template>
   <div class="w-full md:w-4/5 lg:w-3/5 md:mx-auto md:max-w-4xl p-4">
-    <h3 class="font-semibold mt-4 text-xl">
+    <div class="w-full flex flex-col sm:flex-row gap-2">
-      Form Analytics (last 30 days)
+      <div
-    </h3>
+        v-for="(stat, index) in [
+          { label: 'Views', value: totalViews, placeholder: '123' },
+          { label: 'Submissions', value: totalSubmissions, placeholder: '123' },
+          { label: 'Completion Rate', value: completionRate + '%', placeholder: '100%' },
+          { label: 'Average Duration', value: averageDuration, placeholder: '10 seconds' }
+        ]"
+        :key="index"
+        class="border border-gray-300 rounded-lg shadow-sm p-4 w-full mx-auto"
+      >
+        <div class="mb-2 text-sm text-gray-500">
+          {{ stat.label }}
+        </div>
+         
+        <Loader
+          v-if="isLoading"
+          class="h-6 w-6 text-nt-blue"
+        />
+        <span
+          v-else-if="form.is_pro"
+          class="font-medium text-2xl"
+        >
+          {{ stat.value }}
+        </span>
+        <span
+          v-else
+          class="blur-[3px]"
+        >
+          {{ stat.placeholder }}
+        </span>
+      </div>
+    </div>
+
     <form-stats :form="form" />
   </div>
 </template>
@@ -20,4 +51,33 @@ definePageMeta({
 useOpnSeoMeta({
   title: props.form ? "Form Analytics - " + props.form.title : "Form Analytics",
 })
+
+const isLoading = ref(false)
+const totalViews = ref(0)
+const totalSubmissions = ref(0)
+const completionRate = ref(0)
+const averageDuration = ref('-')
+
+onMounted(() => {
+  getCardData()
+})
+
+const getCardData = async() => {
+  if (!props.form || !props.form.is_pro) { return null }
+  isLoading.value = true
+  opnFetch(
+      "/open/workspaces/" +
+      props.form.workspace_id +
+      "/form-stats-details/" +
+      props.form.id,
+  ).then((responseData) => {
+    if (responseData) {
+      totalViews.value = responseData.views ?? 0
+      totalSubmissions.value = responseData.submissions ?? 0
+      completionRate.value = Math.min(100,responseData.completion_rate ?? 0)
+      averageDuration.value = responseData.average_duration ?? '-'
+      isLoading.value = false
+    }
+  })
+}
 </script>