{ "questionnaire": "AI-CAIQ", "version": "v1.1-2026", "framework": "CSA AI Controls Matrix (AICM) v1.1", "vendor": "CMDash, Inc.", "vendorUrl": "https://cmdash.com", "contact": "security@cmdash.com", "dateCompleted": "2026-07-07", "completedBy": "security@cmdash.com", "scope": "How CMDash secures its own use of AI (Anthropic Claude) across 6 internal product features: AI-assisted policy drafting, policy-control link autosuggestion, vendor-questionnaire auto-answer, the daily AI risk narrative, AI executive-summary generation, and AI risk-remediation suggestions. CMDash does not train, fine-tune, or self-host any models — every question is answered against usage of a hosted third-party model, which is why most Model Development Security (MDS) questions are NA. This is a representative subset (52 questions) of AI-CAIQ v1.1's full 320-question set, covering all 18 AICM v1.1 domains (the 17 CSA CCM v4 domains plus the new Model Development Security domain) at a level proportionate to CMDash's actual AI footprint — a handful of narrow backend integrations, not a model-hosting or model-training business.", "answerLegend": { "Yes": "Control is fully implemented with evidence.", "No": "Control is not implemented; remediation status is in notes.", "Partial": "Control is partially implemented; the gap is described in notes.", "NA": "Not applicable — typically because CMDash uses a hosted third-party model rather than training or self-hosting one." }, "sections": [ { "id": "A&A", "title": "Audit & Assurance", "questions": [ { "id": "A&A-01", "question": "Are AI-specific risks included in the organization's security audit program?", "answer": "Partial", "notes": "This questionnaire and its companion internal audit are the first AI-specific review pass. Prior security audits (see /AUDIT.md) covered authz, authn, multi-tenancy/RLS, dependencies, and infra, but not LLM-specific risk classes like prompt injection or output-validation gaps." }, { "id": "A&A-02", "question": "Are findings from AI security reviews tracked to remediation with an owner and target?", "answer": "Yes", "evidence": "Findings from this assessment flow into the same docs/improvement-catalog.md / OPPORTUNITIES.md process used for every other engineering finding, reviewed each sprint." }, { "id": "A&A-03", "question": "Is the AI provider's own audit and compliance posture reviewed before use?", "answer": "Partial", "notes": "Anthropic's public trust and compliance documentation was reviewed informally as part of vendor selection; no formal, recorded vendor-security-assessment exists yet specifically for the AI provider." }, { "id": "A&A-04", "question": "Is this assessment made available to customers under NDA or publicly?", "answer": "Yes", "evidence": "Published at /trust/questionnaires; also available in full via security@cmdash.com." } ] }, { "id": "AIS", "title": "Application & Interface Security", "questions": [ { "id": "AIS-01", "question": "Is output from AI/LLM models validated against a schema before being persisted or displayed?", "answer": "Partial", "evidence": "5 of 6 internal AI services validate Claude's JSON output with zod before use — bounded string lengths, closed enums, hard failure on schema mismatch.", "notes": "One service (risk-remediation suggestions) has no output schema and can persist up to 500 characters of raw, unvalidated model text if JSON parsing fails. Tracked for a fix." }, { "id": "AIS-02", "question": "Are AI-feature API endpoints authenticated?", "answer": "Yes", "evidence": "All 6 AI endpoints sit behind the platform's standard JWT auth guard, same as every other endpoint." }, { "id": "AIS-03", "question": "Are AI-feature endpoints subject to the same role-based authorization as other privileged actions?", "answer": "Yes", "evidence": "All 6 AI controllers now apply an explicit OWNER/ADMIN role guard.", "notes": "Fixed 2026-07-07: risk-remediation generate + accept/reject (ai-suggestions.controller.ts) were the one remaining pair with no role guard; both now require OWNER/ADMIN, matching the other 5 AI controllers." }, { "id": "AIS-04", "question": "Is AI-generated content sanitized before being rendered or emailed, to prevent stored XSS?", "answer": "Yes", "evidence": "The one place AI output reaches hand-built markup outside the frontend framework's auto-escaping (the weekly executive-briefing email) HTML-escapes every AI-generated field before send; the daily narrative service strips HTML from its own output before storage." }, { "id": "AIS-05", "question": "Does an AI feature fail closed rather than silently degrading when the provider is unavailable or unconfigured?", "answer": "Yes", "evidence": "Every AI service checks for a configured provider key at startup and returns a clear unavailable / labeled-mock-mode response rather than pretending to produce real model output." } ] }, { "id": "BCR", "title": "Business Continuity & Operational Resilience", "questions": [ { "id": "BCR-01", "question": "Does the platform degrade gracefully when the AI provider is unavailable?", "answer": "Yes", "evidence": "Same fail-closed behavior as AIS-05; no AI-dependent request errors ungracefully when the provider is down." }, { "id": "BCR-02", "question": "Is there a non-AI fallback path for AI-dependent features?", "answer": "Partial", "notes": "One feature (policy-control link autosuggestion) has a true keyword-match fallback. The other five features simply become unavailable rather than degrading to a non-AI alternative." }, { "id": "BCR-03", "question": "Is the platform's core functionality independent of the AI provider (AI is additive, not load-bearing)?", "answer": "Yes", "notes": "All 6 AI features are opt-in enhancements layered on compliance/risk workflows that function fully without AI enabled." } ] }, { "id": "CCC", "title": "Change Control & Configuration Management", "questions": [ { "id": "CCC-01", "question": "Are AI model versions pinned rather than allowed to float?", "answer": "Yes", "evidence": "Model identifiers are hardcoded per service; a model change is a deliberate code diff and review, not a silent provider-side behavior shift." }, { "id": "CCC-02", "question": "Do system-prompt changes go through the same change-control process as other code changes?", "answer": "Yes", "notes": "Prompts are ordinary source files changed through the normal change process (CMDash currently has a single maintainer, so no second reviewer is required (see /trust/soc2)). There is no additional AI-specific prompt-change gate." }, { "id": "CCC-03", "question": "Are AI-feature configuration flags (enable/disable, sample mode) auditable per change?", "answer": "Partial", "notes": "AI enablement and sample-mode are environment-level configuration, not per-tenant, audit-logged settings." } ] }, { "id": "CEK", "title": "Cryptography, Encryption & Key Management", "questions": [ { "id": "CEK-01", "question": "Is the AI provider API key stored as a secret rather than hardcoded in source?", "answer": "Yes", "evidence": "Repo-wide search confirms no hardcoded provider API keys; only placeholder references in .env.example files and runtime environment configuration." }, { "id": "CEK-02", "question": "Is traffic to the AI provider encrypted in transit?", "answer": "Yes", "notes": "Standard HTTPS via the provider SDK; no custom transport configuration overrides this." }, { "id": "CEK-03", "question": "Is the AI provider API key scoped and rotatable independent of other platform secrets?", "answer": "Yes", "notes": "The AI provider key is its own environment variable, set and rotated independently of database credentials or other provider secrets." } ] }, { "id": "DCS", "title": "Datacenter Security", "questions": [ { "id": "DCS-01", "question": "Does CMDash operate its own datacenters or model-hosting infrastructure for AI inference?", "answer": "NA", "notes": "CMDash calls a hosted third-party model API; inference runs on the provider's infrastructure, not CMDash's. Refer to the AI provider's own compliance documentation for their datacenter controls." }, { "id": "DCS-02", "question": "Is there a documented shared-responsibility understanding for AI-infrastructure security?", "answer": "Partial", "notes": "Understood informally (API-layer security is CMDash's responsibility; model-serving infrastructure is the provider's) but not written down as a formal shared-responsibility matrix specific to AI, unlike the AWS shared-responsibility framing CMDash already documents for cloud infrastructure generally." } ] }, { "id": "DSP", "title": "Data Security & Privacy Lifecycle Management", "questions": [ { "id": "DSP-01", "question": "Is tenant data minimized before being sent to the AI model?", "answer": "Partial", "evidence": "No employee names, email addresses, or end-user PII were found in any traced prompt across all 6 services.", "notes": "Organization name and business-context fields (industry, risk titles/descriptions, policy text, questionnaire text) are sent for several features — this is proportionate to what each feature needs, but it is real tenant business data, not anonymized metrics." }, { "id": "DSP-02", "question": "Is there a documented data-flow map of what tenant data reaches the third-party AI provider, per feature?", "answer": "Yes", "evidence": "This questionnaire, and the internal audit it's drawn from, is that map — all 6 services enumerated with exactly what each interpolates into its prompt. Current as of this assessment's date; not yet a living, auto-checked document.", "notes": "Prior to this assessment, no such map existed." }, { "id": "DSP-03", "question": "Does customer-facing documentation accurately describe what data is sent to the AI model?", "answer": "Yes", "notes": "Fixed 2026-07-07: docs/user/feature-guides/ai-narrative.md previously stated organization name and workspace identifiers were 'not included' in AI model inputs, contradicting the code. The guide was rewritten to accurately describe what's sent (organization name and business-context data; no employee names/emails/end-user PII) and to remove several other claims (a dedicated full-page view, 7-day browsable history, framework/tag/asset-group focus settings, a copy-to-clipboard button, a feedback button) that didn't correspond to any shipped functionality." }, { "id": "DSP-04", "question": "Does the AI provider use customer data to train its models?", "answer": "No", "evidence": "The provider's standard enterprise API terms exclude customer data from model training.", "notes": "CMDash relies on the provider's standard API data-handling terms; there is no CMDash-specific zero-retention agreement in place at this time." } ] }, { "id": "GRC", "title": "Governance, Risk & Compliance", "questions": [ { "id": "GRC-01", "question": "Is there a formal AI governance policy covering all internal AI usage?", "answer": "No", "notes": "Tracked on the roadmap as an 'AI Governance module' (NIST AI RMF + ISO 42001 + EU AI Act) alongside the ISO 42001 framework seed. No standalone internal AI-usage policy exists yet beyond ordinary engineering review." }, { "id": "GRC-02", "question": "Is AI usage inventoried — which features use AI, which model, and what data reaches it?", "answer": "Yes", "evidence": "As of this assessment. See DSP-02.", "notes": "This is a point-in-time inventory, not yet enforced by an automated check that would catch a new AI call site added without updating it." }, { "id": "GRC-03", "question": "Is there a designated owner for AI risk?", "answer": "No", "notes": "No formally designated AI-risk-owner role exists yet. Each AI feature is owned by the engineering team that built it." }, { "id": "GRC-04", "question": "Are AI-specific risks (prompt injection, unbounded consumption, hallucination) tracked in the organization's risk register?", "answer": "No", "notes": "Findings from this assessment are being fed into the standard engineering-improvement process, but a standing 'AI risk' category did not exist prior to this assessment." } ] }, { "id": "HRS", "title": "Human Resources", "questions": [ { "id": "HRS-01", "question": "Are employees trained on responsible AI development or use?", "answer": "No", "notes": "General security-awareness training does not yet include an AI-specific module." }, { "id": "HRS-02", "question": "Do engineers building AI features receive AI-specific secure-development guidance (e.g., output validation, prompt-injection awareness)?", "answer": "Yes", "evidence": "Fixed 2026-07-07 (D2): docs/developers/ai-secure-development.md is a completed guideline for anyone building or modifying a Claude-backed feature, documenting real shipped house convention with file:line citations — DI-seam testability, `parseAndValidate` output-schema checks, server-side referent validation, output escaping, the `AI_SAMPLE_MODE` mock-mode contract, token-metering convention, pinned model constants, `isAvailable()` UI gating, and per-answer AI provenance — each pattern explicitly marked shipped or planned rather than presented as aspirational.", "notes": "The document existing satisfies this question (it's a completed artifact engineers can and do reference), distinct from HRS-01 below, which requires the material to actually be delivered/tracked as training, not merely exist." } ] }, { "id": "IAM", "title": "Identity & Access Management", "questions": [ { "id": "IAM-01", "question": "Are AI-feature endpoints subject to tenant-scoped access control?", "answer": "Yes", "evidence": "Every AI service scopes its data-gathering database queries with an explicit tenant-ID filter; no cross-tenant data was found reaching any prompt in this assessment." }, { "id": "IAM-02", "question": "Is tenant isolation for AI-gathered data backed by database-level enforcement, not just application code?", "answer": "No", "notes": "Database-level row-level security has not been cut over in production (see the same-day internal security audit, finding F3). AI-service tenant isolation currently rests on application-level query filtering only, with no independent database-layer backstop." }, { "id": "IAM-03", "question": "Are AI-feature endpoints covered by automated cross-tenant-authorization tests?", "answer": "Yes", "evidence": "Fixed 2026-07-07 (A5): apps/api/test/ai-cross-tenant-authz.spec.ts covers all 6 AI-backed endpoint groups (narrative, executive-summary, ai-suggestions, policy controls/suggestions, tprm ai-suggest) at the service/controller level with mocked Prisma, asserting NotFoundException/empty results on cross-tenant access AND that the Claude client is never invoked as a side effect of a cross-tenant probe. apps/api/test/ai-controllers-guard-reflection.spec.ts extends A1's guard-metadata-reflection pattern to all 6 AI controllers. Both run in the standard `npm run test` suite. A real-HTTP check also exists (e2e/13-ai-cross-tenant-authz.spec.ts).", "notes": "This closes the test-coverage question asked here. It does not change IAM-02: the e2e HTTP-layer test only runs against a live stack (excluded from CI's unit run, like the rest of e2e/), and the underlying database-layer tenant isolation for AI-gathered data is still application-level query filtering only — that stays open until D5's RLS production cutover actually executes." } ] }, { "id": "IPY", "title": "Interoperability & Portability", "questions": [ { "id": "IPY-01", "question": "Can the AI model provider be swapped without a hard architectural dependency on one vendor?", "answer": "No", "notes": "CMDash is Claude-only today. Multi-provider support (additional model vendors) is a roadmap item — some enterprise buyers require a specific LLM provider by policy." }, { "id": "IPY-02", "question": "Can a tenant export or delete data that was sent to the AI provider on their behalf?", "answer": "Partial", "notes": "Tenant data deletion covers CMDash's own database. Whether any residual data at the AI-provider layer is covered by the same deletion request was not separately verified in this assessment." } ] }, { "id": "IVS", "title": "Infrastructure & Virtualization Security", "questions": [ { "id": "IVS-01", "question": "Does CMDash operate model-serving infrastructure that requires independent hardening?", "answer": "NA", "notes": "Same basis as DCS-01 — inference runs on the provider's managed API, not CMDash-operated infrastructure." } ] }, { "id": "LOG", "title": "Logging & Monitoring", "questions": [ { "id": "LOG-01", "question": "Is AI API usage (tokens, cost) logged for observability?", "answer": "Yes", "evidence": "A usage-metering service records prompt and output token counts per AI call, per tenant, best-effort (a metering failure does not fail the underlying request)." }, { "id": "LOG-02", "question": "Are AI service failures and fallback events logged?", "answer": "Yes", "evidence": "AI services log warnings on missing configuration, output-parse failures, and fallback-path activation." }, { "id": "LOG-03", "question": "Is recorded token/cost usage data used to enforce a limit, not just to observe?", "answer": "Partial", "evidence": "Improved 2026-07-07 (A3): AiBudgetService now closes this loop for real — `recordUsage()` writes actual per-call token counts to a per-tenant Redis day-bucket (ai-budget.service.ts:263-278) and `checkAndReserve()` reads that same counter back (`tokensSpentToday()`, :323-326) to deny the next call once `aiDailyTokenBudget` is reached (:173-188). This is genuine enforcement from recorded usage data, wired into all 6 AI services, not just observation.", "notes": "The token/cost ceiling this enforces is opt-in and defaults to 0 (unlimited) per tenant (`RateLimitConfig.aiDailyTokenBudget`, schema.prisma:2127) — no onboarding or seed path sets a non-zero value today, so out of the box no tenant actually has an active token/cost ceiling until an OWNER/ADMIN sets one via `PATCH /rate-limits`. The separately-enforced request-count ceiling (see TVM-02) is active by default, but that's request volume, not token/cost." } ] }, { "id": "SEF", "title": "Security Incident Management, e-Discovery & Cloud Forensics", "questions": [ { "id": "SEF-01", "question": "Does the incident-response plan include AI-specific scenarios (prompt injection, model data leakage)?", "answer": "Yes", "evidence": "Fixed 2026-07-07 (D3): docs/runbooks/ai-incident-response.md documents 4 AI-specific scenarios end-to-end (detection/containment/comms each) — prompt-injection exploitation, tenant data leakage via prompts, provider (Anthropic) outage/degradation, and runaway spend. Every scenario names the platform-wide `AI_ENABLED=false` kill switch (honored uniformly by all 6 AI services since A3, ai-incident-response.md:17-21) as the common containment tool.", "notes": "This is a new, standalone AI-specific runbook, not folded into the general incident-response plan yet — cross-referencing them is a natural follow-up, not a gap in this answer." }, { "id": "SEF-02", "question": "Is there a defined process for reporting a suspected AI-specific incident (e.g., unexpected AI output disclosing something it shouldn't)?", "answer": "No", "notes": "No dedicated in-product mechanism exists for flagging AI output as a suspected incident (there is no feedback or flagging control on any AI feature today). A suspected AI incident would go through the same general support/security-contact channel as any other report, with no AI-specific triage path yet defined." } ] }, { "id": "STA", "title": "Supply-chain, Transparency & Accountability", "questions": [ { "id": "STA-01", "question": "Is the AI model provider disclosed to customers as a data subprocessor?", "answer": "Yes", "notes": "Fixed 2026-07-07: Anthropic is now listed at /trust/subprocessors with purpose, data categories, region, and legal basis. No live subscribing Controllers exist yet under a signed DPA (Trust Center is still pre-launch), so the 30-day change-notification clause has no one to notify; Legal/Security sign-off on the exact wording is still recommended before the public link goes live, per the Trust Center launch checklist's own gate criteria. A further refinement of this disclosure (finer-grained per-feature data-category breakdown, multi-region processing language, updated SCC/DPA legal-basis citations) is drafted on a separate branch (`claude/anthropic-subprocessor-disclosure-legal-review`) explicitly awaiting legal review and has not merged — what's live today is the basic disclosure described above, nothing more." }, { "id": "STA-02", "question": "Is the AI provider's own security and compliance posture documented anywhere accessible to customers?", "answer": "Partial", "notes": "The AI provider publishes its own public trust and compliance documentation; CMDash's Trust Center does not yet reference it." }, { "id": "STA-03", "question": "Are AI-related data-handling terms (data use, no-training-on-customer-data) reflected in customer-facing contracts or the DPA?", "answer": "Partial", "notes": "This assessment reviewed code and product documentation, not contract language. Whether the DPA specifically addresses AI-subprocessor terms was not verified as part of this pass." } ] }, { "id": "TVM", "title": "Threat & Vulnerability Management", "questions": [ { "id": "TVM-01", "question": "Is the application tested against AI-specific attack techniques (prompt injection, jailbreak) as part of the security-testing program?", "answer": "Partial", "evidence": "Fixed 2026-07-07 (A4 + C4): a shared \"data, not instructions\" framing helper (apps/api/src/common/ai/untrusted-data.ts) wraps every tenant-authored interpolation in a delimited block (with escaping so a forged closing delimiter can't break out) across all 6 AI prompt builders. apps/api/test/ai-prompt-injection.spec.ts is a dedicated adversarial regression suite — one injection-payload test per service, asserting the payload only ever reaches the prompt inside a data block — and runs in the standard `npm run test` CI suite, the first AI-specific security test in the program. docs/security/atlas-ai-threat-checklist.md (MITRE ATLAS v5.6.0) maps 21 applicable techniques against the 6 real AI surfaces with 8 runnable red-team exercises.", "notes": "General SAST/DAST/dependency scanning still doesn't target LLM-specific vulnerability classes, and the ATLAS checklist's red-team exercises are manual run-by-hand procedures today, not scheduled or automated — this is real, tested input-side defense, not a full AI security-testing program." }, { "id": "TVM-02", "question": "Is there a rate limit or cost ceiling on AI API calls to prevent abuse or runaway spend?", "answer": "Yes", "evidence": "Fixed 2026-07-07 (A3): a new AiBudgetService (apps/api/src/common/ai/ai-budget.service.ts) is the single choke point all 6 AI services call immediately before their Claude request — narrative.service.ts:159, executive-summary.service.ts:119, ai-suggestions.service.ts:180, policy-drafting.service.ts:239, policy-link.service.ts:357, questionnaire-ai.service.ts:240. It enforces, per tenant, atomically via Redis: a request-window ceiling (default 1000/hour, ai-budget.service.ts:75-76), a tighter narrative-regenerate cooldown (1/hour, :71-73), and an opt-in daily token budget (:173-188). Usage is queryable via `GET /rate-limits/usage` (rate-limiter.controller.ts:39-43) without consuming quota. The platform-wide `AI_ENABLED=false` kill switch (honored uniformly by all 6 services since A3) remains available as an incident-response backstop regardless of Redis health.", "notes": "The daily token-budget dimension defaults to 0 (unlimited) per tenant and is opt-in (see LOG-03) — the request-count ceiling is what's active for every tenant by default. On a Redis outage the service fails open (logs a warning, allows the call) rather than blocking AI features platform-wide; bounded to the outage window." }, { "id": "TVM-03", "question": "Are free-text fields that feed AI prompts subject to length limits to bound cost and abuse?", "answer": "Partial", "evidence": "Fixed 2026-07-07 (A6): zod `.max()` schemas now bound the three surfaces the original assessment named — PolicyDraftingService.draftPolicy (topic 200 / scope 500 / additionalContext 4000 chars, policy-drafting.service.ts:77-95), RiskService.createRisk/updateRisk (title 200 / description 10000 chars, risk.service.ts:87-122), and VendorQuestionnaireService.upsertAnswers (answerText/evidence/notes, 10000 chars each, questionnaire.service.ts:73-105) — each rejecting with a named-field error on overflow.", "notes": "Not yet covered: ComplianceEvidence.description (compliance-evidence.service.ts:42-63) has no length validation at create time and is interpolated into the questionnaire-ai prompt (wrapped as untrusted data per A4, but not size-bounded) — a real remaining gap, not a rounding error." } ] }, { "id": "UEM", "title": "Universal Endpoint Management", "questions": [ { "id": "UEM-01", "question": "Does Universal Endpoint Management apply to CMDash's AI usage?", "answer": "NA", "notes": "UEM covers managed end-user devices. CMDash's AI usage is entirely server-side API calls with no endpoint-management surface." } ] }, { "id": "MDS", "title": "Model Development Security", "questions": [ { "id": "MDS-01", "question": "Does the organization train or fine-tune its own AI models?", "answer": "No", "notes": "CMDash uses the provider's hosted models exclusively — no in-house training pipeline, training data, or model weights exist to secure, which is why most Model Development Security questions are not applicable." }, { "id": "MDS-02", "question": "Are guardrails implemented to constrain model behavior (system prompts, output schemas, content filters)?", "answer": "Partial", "evidence": "System prompts plus zod output-schema validation exist in 5 of 6 services, with bounded field lengths and closed enums.", "notes": "No dedicated prompt-injection guardrail layer or instruction-hierarchy defense exists; interpolated content is not framed as untrusted data anywhere. See TVM-01." }, { "id": "MDS-03", "question": "Is inference traffic to the model provider authenticated and encrypted?", "answer": "Yes", "notes": "Same basis as CEK-01 and CEK-02." }, { "id": "MDS-04", "question": "Is human review required before AI-generated content persists or reaches parties outside the tenant?", "answer": "Partial", "evidence": "2 of 6 features use a non-persisting-preview pattern: generation and saving are two separate calls with a human decision in between.", "notes": "The other 4 either persist immediately (with post-hoc accept/reject) or reach recipients automatically — including an external, board-adjacent weekly email — with no review gate before send." } ] } ] }