{
  "version": 1,
  "prepared_at": "2026-10-30",
  "publisher": "ShowMeYourSaaS",
  "purpose": "An original evaluation plan. All people, workspaces and events below are fictional test data. No vendor has been benchmarked by this file.",
  "groups": {
    "ai-app-builders": {
      "title": "Build a client brief workspace",
      "scenario": "Build a small app for two fictional studios. Each studio can create and edit its own client briefs. Use test accounts and synthetic data. Complete the same brief and a follow-up change in every candidate.",
      "checks": [
        { "id": "create", "task": "Create a brief with a title and contact field", "expected": "Valid input saves once and appears in the list. Empty titles produce a useful validation message." },
        { "id": "persist", "task": "Reload the app and open the saved brief", "expected": "The saved values persist after reload and after deployment to a test environment." },
        { "id": "workspace", "task": "Try reading and changing another studio's brief", "expected": "The server rejects access to a record outside the signed-in account's workspace, including a direct request." },
        { "id": "failure", "task": "Try saving while the save operation fails", "expected": "The app reports the failure, preserves input and does not show an invented success." },
        { "id": "revision", "task": "Add a due date after the first build", "expected": "The change works for new and existing briefs; the original save and access checks still pass." },
        { "id": "handoff", "task": "Review a repository handoff", "expected": "Record how to get the source, configure required services and reproduce a test deployment. Note anything you could not verify." },
        { "id": "accessibility", "task": "Complete the workflow with a keyboard at a narrow viewport", "expected": "Labels, focus, validation and controls remain usable without a mouse or horizontal page clipping." },
        { "id": "cost", "task": "Record build, revision and running costs separately", "expected": "Record the actual plan, usage, billing period and currency. Keep observed charges separate from estimated future costs." }
      ]
    },
    "analytics": {
      "title": "Follow a signup through to activation",
      "scenario": "Use the same fictional event definitions in each candidate. Four visitors arrive; three sign up; two create a brief after signup. One signup is delivered twice with the same event ID. Compare acquisition reporting and behavioral reporting according to the question each tool is intended to answer.",
      "fixtures": [
        { "id": "e1", "visitor": "a", "event": "page_view", "at": "2026-10-01T10:00:00Z", "source": "community" },
        { "id": "e2", "visitor": "b", "event": "page_view", "at": "2026-10-01T10:01:00Z", "source": "search" },
        { "id": "e3", "visitor": "c", "event": "page_view", "at": "2026-10-01T10:02:00Z", "source": "community" },
        { "id": "e4", "visitor": "d", "event": "page_view", "at": "2026-10-01T10:03:00Z", "source": "direct" },
        { "id": "e5", "visitor": "a", "event": "signup", "at": "2026-10-01T10:04:00Z", "source": "community" },
        { "id": "e5", "visitor": "a", "event": "signup", "at": "2026-10-01T10:04:00Z", "source": "community" },
        { "id": "e6", "visitor": "b", "event": "signup", "at": "2026-10-01T10:05:00Z", "source": "search" },
        { "id": "e7", "visitor": "c", "event": "signup", "at": "2026-10-01T10:06:00Z", "source": "community" },
        { "id": "e8", "visitor": "a", "event": "brief_created", "at": "2026-10-01T10:07:00Z", "source": "community" },
        { "id": "e9", "visitor": "c", "event": "brief_created", "at": "2026-10-01T10:08:00Z", "source": "community" }
      ],
      "expected_after_deduplication": { "unique_visitors": 4, "unique_signups": 3, "activated_visitors": 2, "signup_rate": "3/4", "activation_among_signups": "2/3" },
      "checks": [
        { "id": "definition", "task": "Define arrival, signup and activation", "expected": "Keep the same event names, population, time zone and date window across candidates." },
        { "id": "deduplicate", "task": "Send the repeated signup event", "expected": "Explain and verify deduplication. The fictional fixture contains three unique signed-up visitors, not four." },
        { "id": "identity", "task": "Connect anonymous arrival with signup", "expected": "Document the identity rule and verify one known journey without collecting production customer details." },
        { "id": "funnel", "task": "Reproduce the relevant conversion question", "expected": "The fixture has 4 arrivals, 3 signed-up visitors and 2 later activations. Record which parts your chosen tool and plan can represent." },
        { "id": "source", "task": "Inspect the acquisition source", "expected": "Both activated visitors in this fictional fixture came from community. Explain how attribution is retained or intentionally limited." },
        { "id": "reproduce", "task": "Have another person reproduce the report", "expected": "A second reader can identify the included population and recreate the date range and definitions." },
        { "id": "privacy", "task": "Review collection and access settings", "expected": "Record the fields captured, retention, access and privacy controls before adding real customer activity." },
        { "id": "cost", "task": "Estimate the reporting setup's full cost", "expected": "Record plan features and billable units. Compare current usage with one explicitly labeled higher-usage scenario." }
      ]
    }
  }
}