Better Form Stats (#567)

* Better Form Stats * fix lint * submission timer store in localstorage * Update test case for stats * remove extra code * fix form stats * on restart remove timer * fix resetTimer function name * Improve form timer * Fix timer after form validation error + polish UI --------- Co-authored-by: Julien Nahum <julien@nahum.net>
2024-09-18 22:50:52 +05:30 · 2024-09-18 22:50:52 +05:30 · ceb0648262
parent a057045ef6
commit ceb0648262
14 changed files with 381 additions and 62 deletions
--- a/api/app/Http/Controllers/Forms/FormStatsController.php
+++ b/api/app/Http/Controllers/Forms/FormStatsController.php
@ -3,7 +3,9 @@
 namespace App\Http\Controllers\Forms;

 use App\Http\Controllers\Controller;
+use App\Http\Requests\FormStatsRequest;
 use Carbon\CarbonPeriod;
+use Carbon\CarbonInterval;
 use Illuminate\Http\Request;

 class FormStatsController extends Controller
@ -13,14 +15,14 @@ class FormStatsController extends Controller
        $this->middleware('auth');
    }

-    public function getFormStats(Request $request)
+    public function getFormStats(FormStatsRequest $request)
    {
        $form = $request->form; // Added by ProForm middleware
        $this->authorize('view', $form);

-        $formStats = $form->statistics()->where('date', '>', now()->subDays(29)->startOfDay())->get();
+        $formStats = $form->statistics()->whereBetween('date', [$request->date_from, $request->date_to])->get();
        $periodStats = ['views' => [], 'submissions' => []];
-        foreach (CarbonPeriod::create(now()->subDays(29), now()) as $dateObj) {
+        foreach (CarbonPeriod::create($request->date_from, $request->date_to) as $dateObj) {
            $date = $dateObj->format('d-m-Y');

            $statisticData = $formStats->where('date', $dateObj->format('Y-m-d'))->first();
@ -34,4 +36,26 @@ class FormStatsController extends Controller

        return $periodStats;
    }
+
+    public function getFormStatsDetails(Request $request)
+    {
+        $form = $request->form; // Added by ProForm middleware
+        $this->authorize('view', $form);
+
+        $totalViews = $form->views()->count();
+        $totalSubmissions = $form->submissions_count;
+
+        $averageDuration = \Cache::remember('form_stats_average_duration_' . $form->id, 1800, function () use ($form) {
+            $submissionsWithDuration = $form->submissions()->whereNotNull('completion_time')->count() ?? 0;
+            $totalDuration = $form->submissions()->whereNotNull('completion_time')->sum('completion_time') ?? 0;
+            return $submissionsWithDuration > 0 ? round($totalDuration / $submissionsWithDuration) : null;
+        });
+
+        return [
+            'views' => $totalViews,
+            'submissions' => $totalSubmissions,
+            'completion_rate' => $totalViews > 0 ? round(($totalSubmissions / $totalViews) * 100, 2) : 0,
+            'average_duration' => $averageDuration ? CarbonInterval::seconds($averageDuration)->cascade()->forHumans() : null
+        ];
+    }
 }
--- a/api/app/Http/Controllers/Forms/PublicFormController.php
+++ b/api/app/Http/Controllers/Forms/PublicFormController.php
@ -77,7 +77,7 @@ class PublicFormController extends Controller

        $internal_url = Storage::temporaryUrl($path, now()->addMinutes(5));

-        foreach(config('filesystems.disks.s3.temporary_url_rewrites') as $from => $to) {
+        foreach (config('filesystems.disks.s3.temporary_url_rewrites') as $from => $to) {
            $internal_url = str_replace($from, $to, $internal_url);
        }

@ -89,12 +89,16 @@ class PublicFormController extends Controller
        $form = $request->form;
        $submissionId = false;

+        $submissionData = $request->validated();
+        $completionTime = $request->get('completion_time') ?? null;
+        unset($submissionData['completion_time']); // Remove completion_time from the main data array
+
        if ($form->editable_submissions) {
-            $job = new StoreFormSubmissionJob($form, $request->validated());
+            $job = new StoreFormSubmissionJob($form, $submissionData, $completionTime);
            $job->handle();
            $submissionId = Hashids::encode($job->getSubmissionId());
        } else {
-            StoreFormSubmissionJob::dispatch($form, $request->validated());
+            StoreFormSubmissionJob::dispatch($form, $submissionData, $completionTime);
        }

        return $this->success(array_merge([
--- a/api/app/Http/Requests/FormStatsRequest.php
+++ b/api/app/Http/Requests/FormStatsRequest.php
@ -0,0 +1,31 @@
+<?php
+
+namespace App\Http\Requests;
+
+use Illuminate\Foundation\Http\FormRequest;
+use Carbon\Carbon;
+
+class FormStatsRequest extends FormRequest
+{
+    public function authorize()
+    {
+        return true;
+    }
+
+    public function rules()
+    {
+        return [
+            'date_from' => 'required|date',
+            'date_to' => 'required|date|after_or_equal:date_from',
+        ];
+    }
+
+    public function withValidator($validator)
+    {
+        $validator->after(function ($validator) {
+            if (Carbon::parse($this->date_from)->diffInMonths(Carbon::parse($this->date_to)) > 3) {
+                $validator->errors()->add('date_range', 'Date range exceeds 3 months. Please select a shorter period.');
+            }
+        });
+    }
+}
--- a/api/app/Http/Resources/FormSubmissionResource.php
+++ b/api/app/Http/Resources/FormSubmissionResource.php
@ -23,7 +23,8 @@ class FormSubmissionResource extends JsonResource
        }

        return array_merge([
-            'data' => $this->data
+            'data' => $this->data,
+            'completion_time' => $this->completion_time,
        ], ($this->publiclyAccessed) ? [] : [
            'form_id' => $this->form_id,
            'id' => $this->id
--- a/api/app/Jobs/Form/StoreFormSubmissionJob.php
+++ b/api/app/Jobs/Form/StoreFormSubmissionJob.php
@ -33,7 +33,7 @@ class StoreFormSubmissionJob implements ShouldQueue
     *
     * @return void
     */
-    public function __construct(public Form $form, public array $submissionData)
+    public function __construct(public Form $form, public array $submissionData, public ?int $completionTime = null)
    {
    }

@ -70,11 +70,13 @@ class StoreFormSubmissionJob implements ShouldQueue
        // Create or update record
        if ($previousSubmission = $this->submissionToUpdate()) {
            $previousSubmission->data = $formData;
+            $previousSubmission->completion_time = $this->completionTime;
            $previousSubmission->save();
            $this->submissionId = $previousSubmission->id;
        } else {
            $response = $this->form->submissions()->create([
                'data' => $formData,
+                'completion_time' => $this->completionTime,
            ]);
            $this->submissionId = $response->id;
        }
--- a/api/app/Models/Forms/FormSubmission.php
+++ b/api/app/Models/Forms/FormSubmission.php
@ -11,12 +11,14 @@ class FormSubmission extends Model

    protected $fillable = [
        'data',
+        'completion_time',
    ];

    protected function casts(): array
    {
        return [
            'data' => 'array',
+            'completion_time' => 'integer',
        ];
    }

--- a/api/database/migrations/2024_09_09_180618_add_completion_time_to_form_submissions.php
+++ b/api/database/migrations/2024_09_09_180618_add_completion_time_to_form_submissions.php
@ -0,0 +1,27 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class () extends Migration {
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('form_submissions', function (Blueprint $table) {
+            $table->integer('completion_time')->nullable();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('form_submissions', function (Blueprint $table) {
+            $table->dropColumn('completion_time');
+        });
+    }
+};
--- a/api/routes/api.php
+++ b/api/routes/api.php
@ -146,6 +146,7 @@ Route::group(['middleware' => 'auth:api'], function () {

                Route::middleware('pro-form')->group(function () {
                    Route::get('form-stats/{formId}', [FormStatsController::class, 'getFormStats'])->name('form.stats');
+                    Route::get('form-stats-details/{formId}', [FormStatsController::class, 'getFormStatsDetails'])->name('form.stats-details');
                });
            });
        });
--- a/api/tests/Feature/Forms/FormStatTest.php
+++ b/api/tests/Feature/Forms/FormStatTest.php
@ -39,7 +39,7 @@ it('check formstat chart data', function () {
    }

    // Now check chart data
-    $this->getJson(route('open.workspaces.form.stats', [$workspace->id, $form->id]))
+    $this->getJson(route('open.workspaces.form.stats', [$workspace->id, $form->id]) . '?date_from=' . now()->subDays(29)->format('Y-m-d') . '&date_to=' . now()->format('Y-m-d'))
        ->assertSuccessful()
        ->assertJson(function (\Illuminate\Testing\Fluent\AssertableJson $json) use ($views, $submissions) {
            return $json->whereType('views', 'array')
@ -65,3 +65,37 @@ it('check formstat chart data', function () {
                ->etc();
        });
 });
+
+
+it('checks form stats details', function () {
+    $user = $this->actingAsProUser();
+    $workspace = $this->createUserWorkspace($user);
+    $form = $this->createForm($user, $workspace, []);
+
+    // Create form submissions with varying completion times
+    $form->submissions()->createMany([
+        ['completion_time' => 60],  // 1 minute
+        ['completion_time' => 60],  // 1 minute
+        ['completion_time' => 60],  // 1 minute
+        ['completion_time' => 120], // 2 minutes
+        ['completion_time' => 120], // 2 minutes
+        [] // Incomplete submission
+    ]);
+
+    // Create form views
+    $form->views()->createMany(array_fill(0, 10, []));
+
+    $this->getJson(route('open.workspaces.form.stats-details', [$workspace->id, $form->id]))
+        ->assertSuccessful()
+        ->assertJson(function (\Illuminate\Testing\Fluent\AssertableJson $json) {
+            return $json->has('views')
+                ->has('submissions')
+                ->has('completion_rate')
+                ->has('average_duration')
+                ->where('views', 10)
+                ->where('submissions', 6)
+                ->where('completion_rate', 60)
+                ->where('average_duration', '1 minute 24 seconds')
+                ->etc();
+        });
+});
--- a/client/components/open/forms/FormTimer.vue
+++ b/client/components/open/forms/FormTimer.vue
@ -0,0 +1,57 @@
+<script setup>
+import { ref, onMounted, onUnmounted } from 'vue'
+import { pendingSubmission as pendingSubmissionFun } from "~/composables/forms/pendingSubmission.js"
+
+const props = defineProps({
+  form: { type: Object, required: true }
+})
+
+const pendingSubmission = pendingSubmissionFun(props.form)
+const startTime = ref(null)
+const completionTime = ref(parseInt(pendingSubmission.getTimer() ?? null))
+let timer = null
+
+watch(() => completionTime.value, () => {
+  if (completionTime.value) {
+    pendingSubmission.setTimer(completionTime.value)
+  }
+}, { immediate: true })
+
+const startTimer = () => {
+  if (!startTime.value) {
+    startTime.value = parseInt(pendingSubmission.getTimer() ?? 1)
+    completionTime.value = startTime.value
+    timer = setInterval(() => {
+      completionTime.value += 1
+    }, 1000)
+  }
+}
+
+const stopTimer = () => {
+  if (timer) {
+    clearInterval(timer)
+    timer = null
+    startTime.value = null
+  }
+}
+
+const resetTimer = () => {
+  stopTimer()
+  completionTime.value = null
+}
+
+onMounted(() => {
+  document.addEventListener('input', startTimer)
+})
+
+onUnmounted(() => {
+  document.removeEventListener('input', startTimer)
+  stopTimer()
+})
+
+defineExpose({
+  completionTime,
+  stopTimer,
+  resetTimer
+})
+</script>
--- a/client/components/open/forms/OpenCompleteForm.vue
+++ b/client/components/open/forms/OpenCompleteForm.vue
@ -4,6 +4,11 @@
    class="open-complete-form"
    :style="{ '--font-family': form.font_family }"
  >
+    <FormTimer
+      ref="formTimer"
+      :form="form"
+    />
+
    <link
      v-if="adminPreview && form.font_family"
      rel="stylesheet"
@ -200,6 +205,7 @@
 <script>
 import OpenForm from './OpenForm.vue'
 import OpenFormButton from './OpenFormButton.vue'
+import FormTimer from './FormTimer.vue'
 import VButton from '~/components/global/VButton.vue'
 import FormCleanings from '../../pages/forms/show/FormCleanings.vue'
 import VTransition from '~/components/global/transitions/VTransition.vue'
@ -208,7 +214,7 @@ import clonedeep from "clone-deep"
 import ThemeBuilder from "~/lib/forms/themes/ThemeBuilder.js"

 export default {
-  components: { VTransition, VButton, OpenFormButton, OpenForm, FormCleanings },
+  components: { VTransition, VButton, OpenFormButton, OpenForm, FormCleanings, FormTimer },

  props: {
    form: { type: Object, required: true },
@ -274,8 +280,15 @@ export default {

      if (form.busy) return
      this.loading = true
+
+      // Stop the timer and get the completion time
+      this.$refs.formTimer.stopTimer()
+      const completionTime = this.$refs.formTimer.completionTime
+      form.completion_time = completionTime
+
      // this.closeAlert()
-      form.post('/forms/' + this.form.slug + '/answer').then((data) => {
+      form.post('/forms/' + this.form.slug + '/answer')
+      .then((data) => {
        useAmplitude().logEvent('form_submission', {
          workspace_id: this.form.workspace_id,
          form_id: this.form.id
@ -288,7 +301,8 @@ export default {
            id: this.form.id,
            redirect_target_url: (this.form.is_pro && data.redirect && data.redirect_url) ? data.redirect_url : null
          },
-          submission_data: form.data()
+          submission_data: form.data(),
+          completion_time: completionTime
        })

        if (this.isIframe) {
@ -296,6 +310,7 @@ export default {
        }
        window.postMessage(payload, '*')
        this.pendingSubmission.remove()
+        this.pendingSubmission.removeTimer()

        if (data.redirect && data.redirect_url) {
          window.location.href = data.redirect_url
@ -319,12 +334,14 @@ export default {
          useAlert().error(error.data.message)
        }
        this.loading = false
+        this.$refs.formTimer.startTimer()
        onFailure()
      })
    },
    restart () {
      this.submitted = false
      this.$emit('restarted', true)
+      this.$refs.formTimer.resetTimer() // Reset the timer
    },
    passwordEntered () {
      if (this.passwordForm.password !== '' && this.passwordForm.password !== null) {
--- a/client/components/open/forms/components/FormStats.vue
+++ b/client/components/open/forms/components/FormStats.vue
@ -1,45 +1,60 @@
 <template>
-  <div
-    class="border border-nt-blue-light bg-blue-50 dark:bg-notion-dark-light rounded-md p-4 mb-5 w-full mx-auto mt-4 select-all"
-  >
-    <div
-      v-if="!form.is_pro"
-      class="relative"
-    >
-      <div class="absolute inset-0 z-10">
-        <div class="p-5 max-w-md mx-auto mt-5">
-          <p class="text-center">
-            You need a <pro-tag
-              upgrade-modal-title="Upgrade today to access form analytics"
-              class="mx-1"
-            /> subscription to access your form
-            analytics.
-          </p>
-          <p class="mt-5 text-center">
-            <v-button
-              class="w-full"
-              @click.prevent="subscriptionModalStore.openModal()"
-            >
-              Subscribe
-            </v-button>
-          </p>
-        </div>
-      </div>
-      <img
-        src="/img/pages/forms/blurred_graph.png"
-        alt="Sample Graph"
-        class="mx-auto filter blur-md z-0"
-      >
+  <div>
+    <div class="flex flex-wrap items-end mt-5">
+      <h3 class="flex-grow font-medium text-lg mb-3">
+        Views & Submission History
+      </h3>
+      <DateInput
+        :form="filterForm"
+        name="filter_date"
+        class="flex-1 !mb-0"
+        :date-range="true"
+        :disable-future-dates="true"
+        :disabled="!form.is_pro"
+      />
+    </div>
+    <div
+      class="border border-gray-300 rounded-lg shadow-sm p-4 mb-5 w-full mx-auto mt-4 select-all"
+    >
+      <div
+        v-if="!form.is_pro"
+        class="relative"
+      >
+        <div class="absolute inset-0 z-10">
+          <div class="p-5 max-w-md mx-auto mt-5">
+            <p class="text-center">
+              You need a <pro-tag
+                upgrade-modal-title="Upgrade today to access form analytics"
+                class="mx-1"
+              /> subscription to access your form
+              analytics.
+            </p>
+            <p class="mt-5 text-center">
+              <v-button
+                class="w-full"
+                @click.prevent="subscriptionModalStore.openModal()"
+              >
+                Subscribe
+              </v-button>
+            </p>
+          </div>
+        </div>
+        <img
+          src="/img/pages/forms/blurred_graph.png"
+          alt="Sample Graph"
+          class="mx-auto filter blur-md z-0"
+        >
+      </div>
+      <Loader
+        v-else-if="isLoading"
+        class="h-6 w-6 text-nt-blue mx-auto"
+      />
+      <LineChart
+        v-else
+        :options="chartOptions"
+        :data="chartData"
+      />
    </div>
-    <Loader
-      v-else-if="isLoading"
-      class="h-6 w-6 text-nt-blue mx-auto"
-    />
-    <LineChart
-      v-else
-      :options="chartOptions"
-      :data="chartData"
-    />
  </div>
 </template>

@ -55,7 +70,6 @@ import {
  CategoryScale,
  PointElement,
 } from "chart.js"
-import ProTag from "~/components/global/ProTag.vue"

 ChartJS.register(
  Title,
@ -69,10 +83,7 @@ ChartJS.register(

 export default {
  name: "FormStats",
-  components: {
-    ProTag,
-    LineChart,
-  },
+  components: { LineChart },
  props: {
    form: {
      type: Object,
@ -81,8 +92,12 @@ export default {
  },
  setup() {
    const subscriptionModalStore = useSubscriptionModalStore()
+    const filterForm = useForm({
+      filter_date: null,
+    })
    return {
-      subscriptionModalStore
+      subscriptionModalStore,
+      filterForm
    }
  },
  data() {
@ -119,8 +134,23 @@ export default {
      },
    }
  },
+  watch: {
+    filterForm: {
+      deep: true,
+      handler(newVal) {
+        if(newVal.filter_date && Array.isArray(newVal.filter_date) && newVal.filter_date[0] && newVal.filter_date[1]) {
+          this.getChartData()
+        }
+      }
+    }
+  },
  mounted() {
-    this.getChartData()
+    if (this.form.is_pro) {
+      const toDate = new Date()
+      const fromDate = new Date(toDate)
+      fromDate.setDate(toDate.getDate() - 29)
+      this.filterForm.filter_date = [fromDate.toISOString().split('T')[0], toDate.toISOString().split('T')[0]]
+    }
  },
  methods: {
    getChartData() {
@ -131,6 +161,12 @@ export default {
          this.form.workspace_id +
          "/form-stats/" +
          this.form.id,
+        {
+          params: {
+            date_from: this.filterForm.filter_date[0] ? this.filterForm.filter_date[0].split('T')[0] : null,
+            date_to: this.filterForm.filter_date[1] ? this.filterForm.filter_date[1].split('T')[0] : null,
+          }
+        }
      ).then((statsData) => {
        if (statsData && statsData.views !== undefined) {
          this.chartData.labels = Object.keys(statsData.views)
@ -138,6 +174,9 @@ export default {
          this.chartData.datasets[1].data = statsData.submissions
          this.isLoading = false
        }
+      }).catch((error) => {
+        this.isLoading = false
+        useAlert().error(error.data.message)
      })
    },
  },
--- a/client/composables/forms/pendingSubmission.js
+++ b/client/composables/forms/pendingSubmission.js
@ -7,6 +7,9 @@ export const pendingSubmission = (form) => {
      ? form.form_pending_submission_key + "-" + hash(window.location.href)
      : ""
  })
+  const formPendingSubmissionTimerKey = computed(() => {
+    return formPendingSubmissionKey.value + "-timer"
+  })

  const enabled = computed(() => {
    return form?.auto_save ?? false
@ -28,10 +31,27 @@ export const pendingSubmission = (form) => {
    return pendingSubmission ? JSON.parse(pendingSubmission) : defaultValue
  }

+  const setTimer = (value) => {
+    if (import.meta.server) return
+    useStorage(formPendingSubmissionTimerKey.value).value = value
+  }
+
+  const removeTimer = () => {
+    return setTimer(null)
+  }
+
+  const getTimer = (defaultValue = null) => {
+    if (import.meta.server) return
+    return useStorage(formPendingSubmissionTimerKey.value).value ?? defaultValue
+  }
+
  return {
    enabled,
    set,
    get,
    remove,
+    setTimer,
+    removeTimer,
+    getTimer,
  }
 }
--- a/client/pages/forms/[slug]/show/stats.vue
+++ b/client/pages/forms/[slug]/show/stats.vue
@ -1,8 +1,39 @@
 <template>
  <div class="w-full md:w-4/5 lg:w-3/5 md:mx-auto md:max-w-4xl p-4">
-    <h3 class="font-semibold mt-4 text-xl">
-      Form Analytics (last 30 days)
-    </h3>
+    <div class="w-full flex flex-col sm:flex-row gap-2">
+      <div
+        v-for="(stat, index) in [
+          { label: 'Views', value: totalViews, placeholder: '123' },
+          { label: 'Submissions', value: totalSubmissions, placeholder: '123' },
+          { label: 'Completion Rate', value: completionRate + '%', placeholder: '100%' },
+          { label: 'Average Duration', value: averageDuration, placeholder: '10 seconds' }
+        ]"
+        :key="index"
+        class="border border-gray-300 rounded-lg shadow-sm p-4 w-full mx-auto"
+      >
+        <div class="mb-2 text-sm text-gray-500">
+          {{ stat.label }}
+        </div>
+         
+        <Loader
+          v-if="isLoading"
+          class="h-6 w-6 text-nt-blue"
+        />
+        <span
+          v-else-if="form.is_pro"
+          class="font-medium text-2xl"
+        >
+          {{ stat.value }}
+        </span>
+        <span
+          v-else
+          class="blur-[3px]"
+        >
+          {{ stat.placeholder }}
+        </span>
+      </div>
+    </div>
+
    <form-stats :form="form" />
  </div>
 </template>
@ -20,4 +51,33 @@ definePageMeta({
 useOpnSeoMeta({
  title: props.form ? "Form Analytics - " + props.form.title : "Form Analytics",
 })
+
+const isLoading = ref(false)
+const totalViews = ref(0)
+const totalSubmissions = ref(0)
+const completionRate = ref(0)
+const averageDuration = ref('-')
+
+onMounted(() => {
+  getCardData()
+})
+
+const getCardData = async() => {
+  if (!props.form || !props.form.is_pro) { return null }
+  isLoading.value = true
+  opnFetch(
+      "/open/workspaces/" +
+      props.form.workspace_id +
+      "/form-stats-details/" +
+      props.form.id,
+  ).then((responseData) => {
+    if (responseData) {
+      totalViews.value = responseData.views ?? 0
+      totalSubmissions.value = responseData.submissions ?? 0
+      completionRate.value = Math.min(100,responseData.completion_rate ?? 0)
+      averageDuration.value = responseData.average_duration ?? '-'
+      isLoading.value = false
+    }
+  })
+}
 </script>