[
  {
    "id": "activepieces",
    "name": "activepieces",
    "tagline": "AI agents, MCPs, and AI workflow automation — open-source Zapier alternative.",
    "description": "Activepieces is an open-source workflow automation platform (Zapier alternative) with native support for AI agents and ~400 MCP servers. Workflows are explicit, reviewable, and versioned, making them a natural supervision substrate for production agent deployments that need human-auditable steps.\n",
    "first_released": "2022-12-03",
    "last_updated": "2026-04-23",
    "maturity_status": "stable",
    "published_by": "activepieces",
    "repository_url": "https://github.com/activepieces/activepieces",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "pre_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "scope_creep",
      "context_pollution",
      "supply_chain_attack"
    ],
    "locus_of_control": [
      "environment",
      "human"
    ],
    "implements_techniques": [
      "mcp-workflow",
      "step-audit",
      "branching",
      "version-control"
    ],
    "integration_surfaces": [
      "web_app",
      "http_service"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 21825,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Workflow-automation supervision cell: constrains agent autonomy through explicit, human-editable pipelines rather than free-form agent loops.\n"
  },
  {
    "id": "agent-governance-toolkit",
    "name": "agent-governance-toolkit",
    "tagline": "AI Agent Governance Toolkit — policy enforcement, zero-trust identity, and execution oversight.",
    "description": "Microsoft's AI Agent Governance Toolkit is a reference implementation of zero-trust governance for enterprise AI agents. Provides identity binding, policy engine, execution audit, and approval workflows. Designed as a template for enterprise deployments rather than a drop-in library.\n",
    "first_released": "2026-03-02",
    "last_updated": "2026-04-23",
    "maturity_status": "experimental",
    "published_by": "microsoft",
    "repository_url": "https://github.com/microsoft/agent-governance-toolkit",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "in_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "security_vulnerability",
      "scope_creep",
      "supply_chain_attack"
    ],
    "locus_of_control": [
      "environment",
      "human"
    ],
    "implements_techniques": [
      "zero-trust-identity",
      "policy-engine",
      "execution-audit",
      "approval-workflows"
    ],
    "integration_surfaces": [
      "library",
      "cli"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 1177,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Enterprise governance cell: layers zero-trust identity and policy-engine primitives onto agent deployments, complementing runtime guardrails with organizational controls.\n"
  },
  {
    "id": "agenta",
    "name": "agenta",
    "tagline": "Open-source LLMOps platform: prompt playground, prompt management, LLM evaluation, and observability.",
    "description": "Agenta is an open-source LLMOps platform combining a collaborative prompt playground, prompt versioning, LLM evaluation harness, and observability. Positions non-engineer domain experts as first-class prompt editors alongside developers.\n",
    "first_released": "2023-04-26",
    "last_updated": "2026-04-23",
    "maturity_status": "experimental",
    "published_by": "agenta-ai",
    "repository_url": "https://github.com/Agenta-AI/agenta",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error",
      "scope_creep"
    ],
    "locus_of_control": [
      "environment",
      "human"
    ],
    "implements_techniques": [
      "prompt-playground",
      "prompt-versioning",
      "evals",
      "observability"
    ],
    "integration_surfaces": [
      "http_service",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 4050,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies the collaborative-playground cell: distinguishes itself from langfuse/phoenix by centring non-developer prompt iteration and versioning as the entry point to evaluation.\n"
  },
  {
    "id": "agentic-security",
    "name": "agentic-security",
    "tagline": "Agentic LLM vulnerability scanner and AI red-teaming kit.",
    "description": "Agentic Security is a red-teaming toolkit that drives automated attack agents against LLM endpoints. Ships attack modules (multi-turn jailbreaks, role-play, data exfiltration) and scoring plugins, producing a report of identified weaknesses.\n",
    "first_released": "2024-04-11",
    "last_updated": "2026-02-03",
    "maturity_status": "experimental",
    "published_by": "agentic-security",
    "repository_url": "https://github.com/msoedov/agentic_security",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "security_vulnerability",
      "scope_creep",
      "fabrication"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "multi-turn-attack-agents",
      "jailbreak-probes",
      "red-team-reporting"
    ],
    "integration_surfaces": [
      "cli",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 1848,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Agentic-attacker variant of garak: automates multi-turn attack loops rather than single-prompt probes.\n"
  },
  {
    "id": "agno",
    "name": "agno",
    "tagline": "Build multi-agent systems in pure Python.",
    "description": "Agno (formerly Phidata) is a Python multi-agent framework with pluggable memory and knowledge modules. Canonical Pythonic alternative to CrewAI/AutoGen for agent orchestration, with a focus on speed of iteration.\n",
    "first_released": "2022-05-04",
    "last_updated": "2026-04-23",
    "maturity_status": "stable",
    "published_by": "agno-agi",
    "repository_url": "https://github.com/agno-agi/agno",
    "license": "MPL-2.0",
    "control_paradigm": "prevention",
    "temporal_phase": "pre_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "scope_creep",
      "context_pollution"
    ],
    "locus_of_control": [
      "prompt",
      "environment"
    ],
    "implements_techniques": [
      "multi-agent",
      "memory-modules",
      "knowledge-modules"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 39620,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Python multi-agent framework cell. Complements crewai / autogen by a pure-Python + module-plugin design.\n"
  },
  {
    "id": "ai-infra-guard",
    "name": "AI-Infra-Guard",
    "tagline": "Full-stack AI red-teaming platform for securing AI ecosystems.",
    "description": "Tencent AI-Infra-Guard is a full-stack AI red-teaming platform. Covers model, agent, and infrastructure layers — scans MCP servers, evaluates LLM endpoints, and audits agent stacks for vulnerabilities. Positions itself as an end-to-end AI security posture tool.\n",
    "first_released": "2024-12-25",
    "last_updated": "2026-04-23",
    "maturity_status": "experimental",
    "published_by": "tencent",
    "repository_url": "https://github.com/Tencent/AI-Infra-Guard",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "pre_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "security_vulnerability",
      "supply_chain_attack",
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "mcp-scanner",
      "model-red-team",
      "agent-stack-audit"
    ],
    "integration_surfaces": [
      "http_service"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 3546,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Covers the 'AI SOC' cell: combines multiple scan modalities (model, agent, MCP) behind a single dashboard.\n"
  },
  {
    "id": "ai-red-teaming-playground-labs",
    "name": "AI-Red-Teaming-Playground-Labs",
    "tagline": "AI red-teaming labs: turnkey training environments for attacking AI systems.",
    "description": "Microsoft's AI Red Teaming Playground Labs is a curated set of reproducible training environments for hands-on AI red-teaming education — covers prompt injection, data exfiltration, and agent-exploit scenarios. Used for workforce training and capability benchmarking.\n",
    "first_released": "2025-05-07",
    "last_updated": "2026-02-13",
    "maturity_status": "experimental",
    "published_by": "microsoft",
    "repository_url": "https://github.com/microsoft/AI-Red-Teaming-Playground-Labs",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "security_vulnerability",
      "fabrication",
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "red-team-labs",
      "training-environments",
      "capture-the-flag"
    ],
    "integration_surfaces": [
      "web_app"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 1915,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Training-focused rather than probe-focused: fills the 'red-team curriculum' niche, valuable as a skills development complement to runtime scanners.\n"
  },
  {
    "id": "arize-phoenix",
    "name": "Arize Phoenix",
    "tagline": "Open-source LLM tracing and evaluation platform built on OpenTelemetry.",
    "description": "Open-source LLM and agent observability platform from Arize. Ingests OpenInference/OpenTelemetry traces from any framework, provides a local notebook-friendly UI for span-level inspection, and ships built-in LLM-as-judge evaluators (hallucination, relevance, QA correctness, toxicity, agent-tool-call correctness). Integrates with LangChain, LlamaIndex, CrewAI, AutoGen, and the OpenAI/Anthropic SDKs. Self-hostable container plus an Arize AX managed offering sharing the same data model.\n",
    "first_released": "2023-04-17",
    "last_updated": "2026-04-05",
    "maturity_status": "stable",
    "published_by": "arize-ai",
    "repository_url": "https://github.com/Arize-ai/phoenix",
    "documentation_url": "https://arize.com/docs/phoenix",
    "license": "Elastic-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "logic_error",
      "context_pollution",
      "fabrication"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "trace-capture",
      "openinference-instrumentation",
      "llm-as-judge-eval",
      "dataset-regression-eval",
      "span-annotation"
    ],
    "integration_surfaces": [
      "http_service",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Core project is source-available under Elastic-2.0 and self-hostable via Docker, which makes it a data-residency-friendly observability option. Prompts and completions are the primary payload — redact secrets and PII via the SDK hooks before ingest. OpenInference instrumentation is framework-agnostic; audit auto-instrumentation patches for any that wrap sensitive call sites.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 9394,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies detection × post_generation × fully_autonomous alongside Langfuse and LangSmith but with a tighter OpenTelemetry/ OpenInference standardization posture and a notebook-friendly local mode. Worth modeling as a distinct node because it is the de-facto reference implementation of OpenInference semantic conventions for LLM traces.\n"
  },
  {
    "id": "autogen",
    "name": "AutoGen",
    "tagline": "Multi-agent conversation framework with programmable group chats.",
    "description": "Microsoft Research open-source framework for building LLM applications as conversations between configurable agents (AssistantAgent, UserProxyAgent, GroupChat). Supervision primitives: per-agent system message scoping, conversable-agent turn control, code-execution config that gates whether an agent may run tools or code and in what environment, and GroupChatManager for bounded multi-agent discussion. v0.4 rewrote the core as an actor-model runtime (AgentChat on top of autogen-core).\n",
    "first_released": "2023-09-25",
    "last_updated": "2026-03-30",
    "maturity_status": "stable",
    "published_by": "microsoft",
    "repository_url": "https://github.com/microsoft/autogen",
    "documentation_url": "https://microsoft.github.io/autogen/",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "pre_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "scope_creep",
      "context_pollution"
    ],
    "locus_of_control": [
      "prompt",
      "environment"
    ],
    "implements_techniques": [
      "conversable-agents",
      "group-chat-manager",
      "code-execution-config",
      "actor-runtime",
      "system-message-scoping"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [
      "cihon-stein-2025-autonomy-scoring"
    ],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Open-source (MIT) from Microsoft Research. UserProxyAgent can execute code — configure code_execution_config to run inside a container (Docker) rather than on the host, or disable execution entirely for untrusted prompts. Check compatibility across the 0.2 → 0.4 rewrite when auditing existing integrations; the APIs are not identical.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 57354,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies prevention × pre_generation × fully_autonomous for the conversation-centric multi-agent archetype. Distinct locus from CrewAI: AutoGen scopes behavior via per-agent system messages and turn protocols rather than explicit role/task contracts, which is worth separating in the taxonomy.\n"
  },
  {
    "id": "autorag",
    "name": "AutoRAG",
    "tagline": "Open-source framework for retrieval-augmented generation (RAG) evaluation and optimization.",
    "description": "AutoRAG is an open-source framework that searches for the best RAG pipeline for a given dataset — grid-searches over retriever × reranker × generator combinations and scores each against ground-truth evals. Complements RAG observability tools by focusing on offline pipeline selection.\n",
    "first_released": "2024-01-10",
    "last_updated": "2026-04-21",
    "maturity_status": "experimental",
    "published_by": "marker-inc",
    "repository_url": "https://github.com/Marker-Inc-Korea/AutoRAG",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "context_pollution"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "rag-grid-search",
      "retrieval-metrics",
      "answer-metrics"
    ],
    "integration_surfaces": [
      "library",
      "cli"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 4722,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "RAG-specific evaluation cell: distinct from generic eval harnesses by treating the retriever+generator as the unit of optimization.\n"
  },
  {
    "id": "bandit",
    "name": "Bandit",
    "tagline": "Security linter for Python. Catches common code-level vulnerabilities.",
    "description": "Bandit is a Python security linter from PyCQA: it walks the AST looking for common insecure patterns — hardcoded passwords, use of `eval`/`exec`, weak crypto primitives, shell-injection-prone subprocess calls, insecure deserialization, and so on. It runs from the CLI and in CI as a post_generation security gate. Configuration lives in `pyproject.toml [tool.bandit]`, which the audit detector now recognises.\n",
    "first_released": "2014-09-15",
    "last_updated": "2026-04-15",
    "maturity_status": "stable",
    "published_by": "pycqa",
    "repository_url": "https://github.com/PyCQA/bandit",
    "documentation_url": "https://bandit.readthedocs.io/",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "security_vulnerability"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "ast-walking",
      "security-rule-engine",
      "pattern-matching"
    ],
    "integration_surfaces": [
      "cli",
      "ci_app"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [
      "python"
    ],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Standard dev-tool security posture; Bandit runs locally against source and does not execute target code. Like all rule-based scanners it produces both false positives and false negatives — pair it with a complementary scanner (e.g. semgrep) on regulated codepaths.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 6000,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "editorial_notes": "Added 2026-05-01 to fill the pyproject `[tool.*]` detection path so the audit no longer reports bandit as an unknown tool.\n",
    "inclusion_rationale": "Occupies detection × post_generation × fully_autonomous for security_vulnerability in the Python lane. Distinct from Semgrep because Bandit is Python-specific with hand-tuned rules covering Python's standard library footguns; including it lets the audit detector resolve `[tool.bandit]` blocks.\n"
  },
  {
    "id": "beelzebub",
    "name": "beelzebub",
    "tagline": "Secure low-code honeypot framework with LLM-backed system virtualization.",
    "description": "Beelzebub is a low-code honeypot framework that leverages LLMs to simulate realistic responses for deception engineering. Used by security teams to detect attacker interaction with agent-adjacent infrastructure and capture attack traces.\n",
    "first_released": "2022-05-08",
    "last_updated": "2026-04-22",
    "maturity_status": "experimental",
    "published_by": "beelzebub-labs",
    "repository_url": "https://github.com/beelzebub-labs/beelzebub",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "security_vulnerability",
      "supply_chain_attack"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "llm-honeypots",
      "deception",
      "attack-trace-capture"
    ],
    "integration_surfaces": [
      "http_service",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 1966,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "LLM-honeypot cell: relevant for detecting adversarial activity against agent deployments, complementing vuln scanners (garak, fuzzyai).\n"
  },
  {
    "id": "bifrost",
    "name": "bifrost",
    "tagline": "Fastest enterprise AI gateway with adaptive load balancing, failover, and guardrails.",
    "description": "Maxim's Bifrost is an enterprise AI gateway written in Go. Performs LLM request routing, failover, rate limiting, spend caps, and guardrail enforcement across providers. Benchmarked as significantly faster than LiteLLM for high-throughput deployments.\n",
    "first_released": "2025-03-19",
    "last_updated": "2026-04-23",
    "maturity_status": "experimental",
    "published_by": "maxim-hq",
    "repository_url": "https://github.com/maximhq/bifrost",
    "license": "Apache-2.0",
    "control_paradigm": "prevention",
    "temporal_phase": "in_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "security_vulnerability",
      "scope_creep",
      "supply_chain_attack",
      "cascading_failure"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "llm-proxy",
      "load-balancing",
      "rate-limits",
      "guardrails",
      "spend-caps"
    ],
    "integration_surfaces": [
      "proxy_gateway",
      "http_service"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 4209,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Gateway cell of prevention: enforces runtime policy (rate limit, PII filter, provider failover) as a reverse proxy in front of LLM providers.\n"
  },
  {
    "id": "black",
    "name": "Black",
    "tagline": "Opinionated Python code formatter. Removes style argument from PR review.",
    "description": "Black is the canonical opinionated Python formatter: it rewrites source to a single, deterministic style with no user-tunable formatting options. By collapsing the design space of \"how should this look,\" Black prevents scope creep in code review (style nits) and removes a whole class of agent-introduced cosmetic diffs. Configuration lives in `pyproject.toml [tool.black]`, which the audit detector now recognises.\n",
    "first_released": "2018-03-22",
    "last_updated": "2026-04-15",
    "maturity_status": "stable",
    "published_by": "psf",
    "repository_url": "https://github.com/psf/black",
    "documentation_url": "https://black.readthedocs.io/",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "code-formatting",
      "ast-rewriting",
      "deterministic-style"
    ],
    "integration_surfaces": [
      "cli",
      "ci_app",
      "ide_plugin"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [
      "python"
    ],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Standard dev-tool security posture; no special considerations. Black runs locally, parses the AST, and rewrites source — no network access.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 39000,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "editorial_notes": "Added 2026-05-01 to fill the pyproject `[tool.*]` detection path so the audit no longer reports black as an unknown tool.\n",
    "inclusion_rationale": "Occupies prevention × post_generation × fully_autonomous for scope_creep in the Python lane. Mature workhorse; including it lets the audit detector resolve `[tool.black]` blocks and credit projects for having a deterministic-style gate in place.\n"
  },
  {
    "id": "braintrust",
    "name": "Braintrust",
    "tagline": "Commercial eval-first platform for LLM applications with prompt playground and online scoring.",
    "description": "Hosted eval and observability platform for LLM applications. Combines offline dataset-driven evals, an interactive prompt-and- model comparison playground, online scoring on production traces, and a TypeScript/Python SDK for trace capture. Differentiates from observability-first peers by centering the experiment-and-eval workflow: every prompt or model change is scored against datasets before deployment, and regressions are surfaced at diff time.\n",
    "first_released": "2023-09-01",
    "last_updated": "2026-04-05",
    "maturity_status": "stable",
    "published_by": "braintrust-data",
    "documentation_url": "https://www.braintrust.dev/docs",
    "license": "proprietary",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "logic_error",
      "context_pollution",
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "dataset-regression-eval",
      "llm-as-judge-eval",
      "prompt-playground",
      "online-scoring",
      "trace-capture"
    ],
    "integration_surfaces": [
      "http_service",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Proprietary hosted SaaS; prompts, completions, and datasets are sent to Braintrust's backend by default. Enterprise plans offer on-prem proxy and VPC-peered deployment options. Dataset storage contains the training data used for evals — classify accordingly if production traffic is ingested. Audit redaction hooks before enabling capture on traffic containing secrets or customer data.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "contributors": [],
    "inclusion_rationale": "Occupies detection × post_generation × fully_autonomous as the eval-first commercial platform. Worth modeling as a distinct node because its center of gravity is the offline eval + prompt- playground workflow rather than observability traces, which represents a different supervision posture from LangSmith/Langfuse/ Phoenix even though the facet cell is shared.\n"
  },
  {
    "id": "browser-mcp",
    "name": "mcp",
    "tagline": "Browser MCP — an MCP server that lets AI applications control your browser.",
    "description": "BrowserMCP is a community MCP server that exposes the running user's browser (via a browser extension) to AI applications. Differs from headless Playwright by running against the user's real logged-in session, which raises distinct supervision concerns.\n",
    "first_released": "2025-03-28",
    "last_updated": "2025-04-24",
    "maturity_status": "at_risk",
    "published_by": "browsermcp",
    "repository_url": "https://github.com/BrowserMCP/mcp",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "scope_creep",
      "security_vulnerability"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "mcp-server",
      "browser-extension-bridge",
      "real-session-access"
    ],
    "integration_surfaces": [
      "mcp_server"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 6384,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Real-browser (non-headless) MCP cell: important point of contrast with playwright-mcp because the attack surface is the user's live session.\n"
  },
  {
    "id": "browser-use",
    "name": "browser-use",
    "tagline": "Make websites accessible for AI agents.",
    "description": "Browser-use is the widely-adopted Python library that wraps Playwright with an LLM-ready DOM abstraction. Canonical pattern for constraining LLM action space to parseable/interactive DOM elements. Heavily used as a building block in browser-agent stacks.\n",
    "first_released": "2024-10-31",
    "last_updated": "2026-04-21",
    "maturity_status": "stable",
    "published_by": "browser-use",
    "repository_url": "https://github.com/browser-use/browser-use",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "post_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "scope_creep",
      "security_vulnerability"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "dom-abstraction",
      "playwright-wrapper",
      "llm-actionable-elements"
    ],
    "integration_surfaces": [
      "library",
      "web_app"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 89651,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Canonical 'DOM-only' browser-use cell — most other browser agents build on top of or differentiate from it.\n"
  },
  {
    "id": "comfyui",
    "name": "ComfyUI",
    "tagline": "The most powerful and modular diffusion model GUI, API, and backend.",
    "description": "ComfyUI is the node-based GUI/backend for diffusion models. Its node-graph execution model enables reproducible pipelines and is a reference for how generative workflows can be structured and supervised end-to-end.\n",
    "first_released": "2023-01-17",
    "last_updated": "2026-04-23",
    "maturity_status": "stable",
    "published_by": "comfy",
    "repository_url": "https://github.com/comfyanonymous/ComfyUI",
    "license": "GPL-3.0",
    "control_paradigm": "prevention",
    "temporal_phase": "pre_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "node-graph-execution",
      "reproducible-workflows"
    ],
    "integration_surfaces": [
      "web_app",
      "desktop_app"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 109742,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Included as the canonical node-graph-generative-pipeline — relevant as a reference for supervising multimodal/diffusion agent stacks.\n"
  },
  {
    "id": "confident-ai-deepteam",
    "name": "DeepTeam",
    "tagline": "Open-source LLM red-teaming framework from Confident AI with 40+ attack vulnerabilities.",
    "description": "Open-source red-teaming framework for LLM applications, from the authors of DeepEval. Ships a library of vulnerabilities (prompt injection, jailbreak, PII leakage, bias, toxicity, misinformation, excessive agency, unauthorized actions) and attack enhancement strategies (multi-turn, roleplay, encoding, linguistic obfuscation) that stress-test an LLM under test via programmatic attack agents. Findings are emitted as scored reports suitable for pre-deployment gating and continuous-evaluation harnesses.\n",
    "first_released": "2024-11-01",
    "last_updated": "2026-04-01",
    "maturity_status": "stable",
    "published_by": "confident-ai",
    "repository_url": "https://github.com/confident-ai/deepteam",
    "documentation_url": "https://trydeepteam.com",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "security_vulnerability",
      "supply_chain_attack",
      "fabrication"
    ],
    "locus_of_control": [
      "environment",
      "prompt"
    ],
    "implements_techniques": [
      "red-team-attacks",
      "vulnerability-scoring",
      "multi-turn-attack-strategies",
      "llm-as-judge-eval"
    ],
    "integration_surfaces": [
      "library",
      "cli"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Open-source (Apache-2.0). The framework generates adversarial prompts at scale — intended strictly for systems under test. Calls the model-under-test plus a judge model; budget token spend accordingly. Attack corpora evolve with upstream releases; pin versions for reproducible audit evidence.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 1566,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies detection × post_generation × fully_autonomous with a red-team-specialist locus that no other modeled node covers. Adversarial pre-deployment scanning is analytically distinct from trace-capture observability — it proactively synthesizes attack inputs rather than sampling production traffic — and maps cleanly onto the security_vulnerability and supply_chain_attack failure modes.\n"
  },
  {
    "id": "context-hub",
    "name": "Context Hub",
    "tagline": "Up-to-date, curated API docs for coding agents.",
    "description": "CLI and protocol that gives coding agents curated, versioned, LLM-optimized API documentation. Primitives: docs, skills, annotations, feedback. Addresses the training-data-staleness root cause of fabrication and obsolescence at the pre-generation phase.\n",
    "first_released": "2026-03-09",
    "last_updated": "2026-04-15",
    "maturity_status": "experimental",
    "published_by": "deeplearning-ai",
    "repository_url": "https://github.com/andrewyng/context-hub",
    "documentation_url": "https://deeplearning.ai/context-hub",
    "license": "Apache-2.0",
    "control_paradigm": "prevention",
    "temporal_phase": "pre_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "fabrication",
      "obsolescence"
    ],
    "locus_of_control": [
      "context"
    ],
    "implements_techniques": [
      "doc-grounded-rag",
      "skill-library"
    ],
    "integration_surfaces": [
      "mcp_server",
      "cli"
    ],
    "composes_with": [
      "pydantic-ai",
      "e2b"
    ],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Open-source, maintained by DeepLearning.AI. Docs delivered via CLI under user control. Consider pinning the CLI version to avoid supply-chain drift.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 13025,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "editorial_notes": "Added 2026-03-10 (from user's v0.1 spec). Fast-moving; revisit maturity_status at 90-day checkpoint.\n",
    "inclusion_rationale": "Occupies a previously unfilled cell in the grid (prevention × pre_generation × fully_autonomous, targeting obsolescence). Directly validates the \"prevention-first\" positioning at the context facet.\n"
  },
  {
    "id": "context7",
    "name": "Context7",
    "tagline": "Up-to-date, version-pinned library documentation for coding agents — delivered as an MCP server.",
    "description": "Context7 is Upstash's MCP server that fetches current, version-pinned documentation for the libraries an agent is about to use, then injects it into the agent's context window before generation. Targets the same training-data-staleness root cause as Andrew Ng's ContextHub but ships as a hosted service plus open-source MCP client. Practitioners install the MCP server in their coding agent and invoke `use context7` in the prompt to ground generation in real, current API surfaces — guarding against fabricated APIs and obsolete-syntax suggestions.\n",
    "first_released": "2025-04-01",
    "last_updated": "2026-05-01",
    "maturity_status": "stable",
    "published_by": "upstash",
    "repository_url": "https://github.com/upstash/context7",
    "documentation_url": "https://context7.com",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "pre_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "fabrication",
      "obsolescence"
    ],
    "locus_of_control": [
      "context"
    ],
    "implements_techniques": [
      "doc-grounded-rag",
      "version-pinned-docs",
      "mcp-context-injection"
    ],
    "integration_surfaces": [
      "mcp_server",
      "cli"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [
      "python",
      "javascript",
      "typescript",
      "go",
      "rust"
    ],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "MCP server fetches docs from Context7's hosted index. Practitioners who can't accept third-party fetches can self-host the server — the source is MIT-licensed. No code execution; documentation strings are inserted into the prompt only.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 54254,
    "stars_updated_at": "2026-05-01",
    "contributors": [],
    "editorial_notes": "Added 2026-05-01 (v0.5.1) following the Stage 4 quality-gate enrichment pass that surfaced it from the long tail. Motivating example for that workstream.\n",
    "inclusion_rationale": "Occupies the same prevention × pre_generation × fully_autonomous cell as ContextHub, with the same locus_of_control (context). Distinct from ContextHub by being a hosted, MCP-native service with broader language coverage; both belong in the catalog because they're structurally different bets on the doc-grounding problem (hosted index vs. self-hosted skill library). Surfaced by the v0.5.1 long-tail enrichment pass (54k stars, originally buried at row 2,488 of 2,675 candidates because it appeared in only 1 awesome-list).\n"
  },
  {
    "id": "copilotkit",
    "name": "CopilotKit",
    "tagline": "The frontend stack for agents and generative UI — React and Angular.",
    "description": "CopilotKit is a React/Angular framework for embedding AI agents into web UIs. Exposes typed actions, generative UI components, and HITL approval UI primitives. Positions the frontend as a first-class supervision layer where humans confirm or adjust agent-proposed actions.\n",
    "first_released": "2023-06-19",
    "last_updated": "2026-04-23",
    "maturity_status": "stable",
    "published_by": "copilotkit",
    "repository_url": "https://github.com/CopilotKit/CopilotKit",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "in_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "scope_creep",
      "context_pollution"
    ],
    "locus_of_control": [
      "environment",
      "human"
    ],
    "implements_techniques": [
      "generative-ui",
      "hitl-approval",
      "typed-actions"
    ],
    "integration_surfaces": [
      "web_app"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 30389,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Frontend-HITL cell: materializes the human-in-the-loop pattern as reusable React components, complementing backend guardrails.\n"
  },
  {
    "id": "coverage-py",
    "name": "Coverage.py",
    "tagline": "Code coverage measurement for Python. Detects test-manipulation by surfacing uncovered code.",
    "description": "Coverage.py instruments Python execution to record which lines and branches are exercised by a test run, then reports per-file and per-line coverage. By surfacing uncovered code it makes test_manipulation (deleted tests, skipped assertions, \"tests pass but exercise nothing\") visible, and exposes incomplete_execution where new code ships without any tests at all. Configuration lives in `pyproject.toml [tool.coverage.*]`, which the audit detector now recognises.\n",
    "first_released": "2004-12-12",
    "last_updated": "2026-04-15",
    "maturity_status": "stable",
    "published_by": "nedbat",
    "repository_url": "https://github.com/nedbat/coveragepy",
    "documentation_url": "https://coverage.readthedocs.io/",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "test_manipulation",
      "incomplete_execution"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "line-coverage-instrumentation",
      "branch-coverage-instrumentation",
      "trace-hooking",
      "coverage-reporting"
    ],
    "integration_surfaces": [
      "cli",
      "ci_app",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [
      "python"
    ],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Standard dev-tool security posture; Coverage.py uses sys.settrace / sys.monitoring to instrument the Python interpreter, so it should not be enabled in production paths. Output reports may include source snippets — treat coverage artefacts with the same access controls as the source repo.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 3000,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "editorial_notes": "Added 2026-05-01 to fill the pyproject `[tool.*]` detection path so the audit no longer reports coverage.py as an unknown tool.\n",
    "inclusion_rationale": "Occupies detection × post_generation × fully_autonomous for test_manipulation in the Python lane — distinct from pytest because the signal is \"what was *not* run,\" not \"what failed.\" Including it lets the audit detector resolve `[tool.coverage.*]` blocks and credit projects with a coverage-gating discipline.\n"
  },
  {
    "id": "coze-loop",
    "name": "coze-loop",
    "tagline": "Full-lifecycle AI agent management platform with debugging, evaluation, and monitoring.",
    "description": "Coze Loop (from ByteDance's Coze team) is a full-lifecycle AI agent management platform with debugging, evaluation, trajectory inspection, and production monitoring. Deployable standalone or integrated with the Coze agent studio.\n",
    "first_released": "2025-06-24",
    "last_updated": "2026-04-22",
    "maturity_status": "stable",
    "published_by": "coze-dev",
    "repository_url": "https://github.com/coze-dev/coze-loop",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "scope_creep",
      "incomplete_execution",
      "context_pollution",
      "cascading_failure"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "agent-trajectory-debug",
      "evals",
      "monitoring"
    ],
    "integration_surfaces": [
      "web_app",
      "http_service"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 5427,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Adds the agent-lifecycle-specific observability + eval cell: trajectory replays and step-level debugging across multi-step workflows. Distinct from opik/langfuse by being agent-first rather than LLM-call-first.\n"
  },
  {
    "id": "crewai",
    "name": "CrewAI",
    "tagline": "Role-based multi-agent framework with task contracts and process flows.",
    "description": "Open-source Python framework for building multi-agent systems by declaring Agents (role, goal, backstory), Tasks (description, expected output, assigned agent), and a Process (sequential or hierarchical execution). Supervision primitives: role/goal contracts that scope each agent's remit, structured task outputs, a manager-agent hierarchical process for delegation control, and explicit tool allow-lists per agent. Distinct from conversation-centric frameworks in that roles and task boundaries are first-class.\n",
    "first_released": "2023-12-28",
    "last_updated": "2026-04-05",
    "maturity_status": "stable",
    "published_by": "crewai-inc",
    "repository_url": "https://github.com/crewAIInc/crewAI",
    "documentation_url": "https://docs.crewai.com",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "pre_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "scope_creep",
      "context_pollution"
    ],
    "locus_of_control": [
      "prompt",
      "context"
    ],
    "implements_techniques": [
      "role-goal-contract",
      "task-specification",
      "hierarchical-process",
      "tool-allow-list",
      "manager-agent-delegation"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Open-source (MIT) Python library. Agent tools are plain Python callables with whatever capabilities the integrator grants — tool allow-lists scope which agent can invoke which tool but do not sandbox execution. CrewAI Enterprise is a separate hosted product with its own data-handling posture; audit that separately before sending proprietary code to it.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 49633,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies prevention × pre_generation × fully_autonomous for the role-based multi-agent archetype. Role and task contracts are pre-generation scoping primitives: they constrain what each agent can legitimately attempt before any token is emitted.\n"
  },
  {
    "id": "daytona",
    "name": "Daytona",
    "tagline": "Secure AI sandbox workspaces with sub-second start and stateful dev environments.",
    "description": "Open-core sandbox platform providing secure, stateful workspaces for AI agents. Supervision primitives: sub-second sandbox creation, per-workspace filesystem and network isolation, snapshot/fork for branching trajectories, and a declarative devcontainer-style workspace config. Positioned as infrastructure for fully_autonomous coding agents that need reproducible, disposable execution environments.\n",
    "first_released": "2024-02-15",
    "last_updated": "2026-04-08",
    "maturity_status": "stable",
    "published_by": "daytona-io",
    "repository_url": "https://github.com/daytonaio/daytona",
    "documentation_url": "https://www.daytona.io/docs",
    "license": "AGPL-3.0",
    "control_paradigm": "recovery",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "supply_chain_attack",
      "security_vulnerability",
      "logic_error"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "sandboxed-execution",
      "workspace-snapshot-fork",
      "ephemeral-filesystem",
      "resource-limiting"
    ],
    "integration_surfaces": [
      "cli",
      "http_service"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Open-core (AGPL core + hosted control plane). Self-hosted deployments require hardening of the orchestration layer and careful review of runtime-class settings for tenant isolation. Hosted tier inherits typical SaaS tenancy and data-residency questions — review posture before running sensitive code.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 72378,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies recovery × post_generation × fully_autonomous. Distinctive primitive is workspace snapshot/fork, which enables branching agent trajectories and rollback — a supervision capability beyond pure per-call sandbox isolation.\n"
  },
  {
    "id": "deepeval",
    "name": "DeepEval",
    "tagline": "The open-source LLM evaluation framework for testing and evaluating LLM applications across 20+ custom and researched metrics.",
    "description": "DeepEval is an open-source framework for unit testing and evaluating LLM applications. It provides 20+ research-backed evaluation metrics including hallucination detection, correctness, answer relevancy, faithfulness, contextual recall/precision, and prompt injection detection. Developers can integrate it into CI/CD pipelines for continuous evaluation of LLM outputs. Supports custom metrics, chain-of-thought evaluation, and synthetic data generation for test cases.",
    "first_released": "2023-08-10",
    "last_updated": "2026-04-22",
    "maturity_status": "experimental",
    "repository_url": "https://github.com/confident-ai/deepeval",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [],
    "integration_surfaces": [
      "library",
      "cli"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 14938,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies detection × post_generation × full_hitl as a unit-testing harness for LLM applications: developers write pytest-style assertions over 20+ research-backed metrics (faithfulness, contextual recall/precision, hallucination, G-Eval chain-of-thought) and gate CI on them. Distinguished from observability-first peers (langfuse, langsmith) by centering test-case authoring rather than trace capture, and from braintrust by being open-source and Pytest-native. Useful as the canonical CI-gate exemplar for the post-generation evaluation cell.\n"
  },
  {
    "id": "dependabot",
    "name": "Dependabot",
    "tagline": "Automated dependency-update PRs that flag outdated and vulnerable packages.",
    "description": "Automated dependency maintenance service integrated into GitHub. Monitors manifests (package.json, requirements.txt, go.mod, Cargo.toml, etc.) for outdated or CVE-affected versions and opens pull requests that bump to safe releases, with release notes and compatibility scores attached. Runs as a scheduled GitHub-hosted bot plus the open-source dependabot-core engine; PRs are reviewed and merged by humans, so mitigation is graduated-HITL even though scanning itself is autonomous.\n",
    "first_released": "2015-05-26",
    "last_updated": "2026-04-15",
    "maturity_status": "stable",
    "published_by": "github",
    "repository_url": "https://github.com/dependabot/dependabot-core",
    "documentation_url": "https://docs.github.com/en/code-security/dependabot",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "obsolescence",
      "supply_chain_attack",
      "security_vulnerability"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "dependency-version-diff",
      "cve-advisory-matching",
      "automated-pull-request",
      "compatibility-score"
    ],
    "integration_surfaces": [
      "ci_app"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [
      "npm",
      "pypi",
      "maven",
      "go-modules",
      "cargo",
      "bundler",
      "composer",
      "nuget",
      "docker"
    ],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Free for public and private GitHub repos; the hosted service runs in GitHub's infrastructure with repo read/write scopes. Dependabot-core itself is MIT-licensed and can be self-hosted. Fix PRs are signed by the Dependabot bot; review auto-merge policies carefully to avoid rubber-stamping malicious upstream releases masquerading as patch bumps.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 5546,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies detection x post_generation x graduated_hitl for the obsolescence failure mode: once code has shipped with a given pinned version, Dependabot watches the advisory feed and proposes upgrades for a human to merge. Distinctive for being the default, no-configuration-required option on GitHub, which makes it the most-deployed mitigation for stale dependencies in practice. Complements Renovate (same cell, more configurable) and context-hub (pre_generation prevention via fresh docs).\n"
  },
  {
    "id": "docetl",
    "name": "docetl",
    "tagline": "LLM-powered data processing with automatic pipeline optimization.",
    "description": "DocETL (UC Berkeley EPIC Lab) is an LLM-powered data-processing system. Automatically optimizes extraction pipelines with cost-quality trade-offs, with a focus on reliability on complex document extraction tasks.\n",
    "first_released": "2024-07-09",
    "last_updated": "2026-03-27",
    "maturity_status": "experimental",
    "published_by": "ucb-epic",
    "repository_url": "https://github.com/ucbepic/docetl",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "pipeline-optimizer",
      "llm-etl"
    ],
    "integration_surfaces": [
      "library",
      "cli"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 3711,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "LLM-pipeline-optimization cell — complements autorag (RAG pipeline search) for the general ETL case.\n"
  },
  {
    "id": "e2b",
    "name": "E2B",
    "tagline": "Secure sandboxes for AI-generated code execution.",
    "description": "Cloud-hosted sandboxed runtime for executing AI-generated code with isolation, snapshotting, and fine-grained resource limits. Per-session ephemeral environments with filesystem and network controls.\n",
    "first_released": "2023-07-01",
    "last_updated": "2026-04-01",
    "maturity_status": "stable",
    "published_by": "e2b",
    "repository_url": "https://github.com/e2b-dev/e2b",
    "documentation_url": "https://e2b.dev/docs",
    "license": "Apache-2.0",
    "control_paradigm": "recovery",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "supply_chain_attack",
      "security_vulnerability",
      "logic_error"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "sandboxed-execution",
      "resource-limiting",
      "ephemeral-filesystem"
    ],
    "integration_surfaces": [
      "library",
      "http_service"
    ],
    "composes_with": [
      "context-hub"
    ],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Hosted sandbox provider. Review data-residency and tenancy isolation posture for sensitive workloads. Local/on-prem alternatives (Daytona, Modal, Firecracker) are worth evaluating for high-trust environments.\n",
    "signed_manifest": true,
    "openssf_scorecard_score": null,
    "stars": 11878,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Canonical recovery × post_generation × fully_autonomous exemplar. Sandboxed execution is a load-bearing supervision primitive for autonomous coding agents.\n"
  },
  {
    "id": "eslint",
    "name": "ESLint",
    "tagline": "Pluggable JavaScript/TypeScript linter and code-quality enforcer.",
    "description": "ESLint is the canonical JavaScript and TypeScript linter: it parses source to an AST and runs configurable rule plugins covering logic bugs (unused vars, unreachable code, mistaken equality), style consistency (scope_creep on review), and a growing set of security rules (eslint-plugin-security, no-eval). It runs from the CLI, in pre-commit, in CI, and as a language-server in editors. Configuration lives in `eslint.config.js` or `package.json` `eslintConfig`, which the audit detector now recognises.\n",
    "first_released": "2013-06-21",
    "last_updated": "2026-04-15",
    "maturity_status": "stable",
    "published_by": "eslint",
    "repository_url": "https://github.com/eslint/eslint",
    "documentation_url": "https://eslint.org/docs/latest/",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "logic_error",
      "scope_creep",
      "security_vulnerability"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "lint-rule-engine",
      "ast-walking",
      "plugin-architecture",
      "autofix"
    ],
    "integration_surfaces": [
      "cli",
      "ci_app",
      "ide_plugin"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [
      "javascript",
      "typescript"
    ],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Standard dev-tool security posture; ESLint executes plugin code loaded from `node_modules`, so a malicious lint plugin can run with developer privileges — pin plugin versions and prefer audited configurations (e.g. `eslint:recommended`, `@typescript-eslint`).\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 25000,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "editorial_notes": "Added 2026-05-01 to fill the package.json detection path so the audit no longer reports eslint as an unknown tool.\n",
    "inclusion_rationale": "Occupies detection × post_generation × fully_autonomous for logic_error, scope_creep, and security_vulnerability in the JS/TS lane. Mature workhorse; missing it leaves the audit detector unable to recognise the single most common JavaScript linting gate present in nearly every modern Node project.\n"
  },
  {
    "id": "flowise",
    "name": "Flowise",
    "tagline": "Build AI agents, visually.",
    "description": "FlowiseAI is a low-code builder for LLM agent workflows with a drag-and-drop UI. Built on LangChain primitives and packaged as a self-hostable service for non-coders to compose agent pipelines.\n",
    "first_released": "2023-03-31",
    "last_updated": "2026-04-23",
    "maturity_status": "stable",
    "published_by": "flowise-ai",
    "repository_url": "https://github.com/FlowiseAI/Flowise",
    "license": "Apache-2.0",
    "control_paradigm": "prevention",
    "temporal_phase": "pre_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "scope_creep",
      "context_pollution"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "drag-and-drop-builder",
      "self-hosted",
      "langchain-primitives"
    ],
    "integration_surfaces": [
      "web_app",
      "http_service"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 52185,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Low-code agent builder cell. Distinguished from langflow by being TypeScript/Node-first.\n"
  },
  {
    "id": "fuzzyai",
    "name": "FuzzyAI",
    "tagline": "A powerful tool for automated LLM fuzzing from CyberArk.",
    "description": "CyberArk FuzzyAI is an automated LLM fuzzer: evolves adversarial prompts over many generations to find vulnerabilities in deployed LLM applications. Integrates with popular model providers and produces reproducible attack traces.\n",
    "first_released": "2024-12-03",
    "last_updated": "2026-02-06",
    "maturity_status": "experimental",
    "published_by": "cyberark",
    "repository_url": "https://github.com/cyberark/FuzzyAI",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "security_vulnerability",
      "fabrication"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "evolutionary-prompt-fuzzing",
      "jailbreak-search",
      "reproducible-traces"
    ],
    "integration_surfaces": [
      "cli",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 1337,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Fuzz-based branch of LLM vulnerability scanning — adds evolutionary prompt search alongside the fixed-probe scanners like garak.\n"
  },
  {
    "id": "garak",
    "name": "garak",
    "tagline": "The LLM vulnerability scanner.",
    "description": "NVIDIA garak is an open-source LLM vulnerability scanner: applies a library of probes (jailbreaks, prompt injection, data leakage, malware generation, misinformation) against any LLM endpoint and reports per-probe attack success rates. Models the NVIDIA NeMo Guardrails adversarial test plane.\n",
    "first_released": "2023-05-10",
    "last_updated": "2026-04-21",
    "maturity_status": "stable",
    "published_by": "nvidia",
    "repository_url": "https://github.com/NVIDIA/garak",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "security_vulnerability",
      "fabrication",
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "llm-probes",
      "attack-success-rate-metrics",
      "detect-generators"
    ],
    "integration_surfaces": [
      "cli",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 7614,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Canonical LLM-vuln-scanner: complements llm-guard/rebuff by running at assessment time rather than at request time.\n"
  },
  {
    "id": "giskard",
    "name": "giskard-oss",
    "tagline": "Open-source evaluation and testing library for LLM agents and ML models.",
    "description": "Giskard is an open-source testing library originally for tabular ML, extended to cover LLM apps and agents. Runs pre-defined robustness, bias, and hallucination tests and generates regression reports. Used in CI to gate ML/LLM deployments.\n",
    "first_released": "2022-03-06",
    "last_updated": "2026-04-23",
    "maturity_status": "stable",
    "published_by": "giskard",
    "repository_url": "https://github.com/Giskard-AI/giskard-oss",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "security_vulnerability",
      "fabrication",
      "logic_error",
      "scope_creep",
      "test_manipulation"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "robustness-tests",
      "bias-tests",
      "hallucination-tests",
      "ci-gates"
    ],
    "integration_surfaces": [
      "library",
      "cli",
      "ci_app"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 5295,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "ML-lineage evaluation cell: extends classical ML testing primitives to LLMs, differentiating from LLM-native tools like deepeval.\n"
  },
  {
    "id": "guardrails-ai",
    "name": "Guardrails AI",
    "tagline": "Output validation framework with a catalog of validators and reask-on-failure loops.",
    "description": "Python framework that wraps LLM calls and validates outputs against a declarative spec (Rail/XML or Pydantic). Ships a hub of reusable validators for PII, toxicity, topic, schema, JSON shape, and custom regexes. On validation failure, re-asks the model or raises. Primarily a post-generation detect-and-retry layer, not a decoding-time constraint.\n",
    "first_released": "2023-02-01",
    "last_updated": "2026-03-20",
    "maturity_status": "stable",
    "published_by": "guardrails-ai",
    "repository_url": "https://github.com/guardrails-ai/guardrails",
    "documentation_url": "https://www.guardrailsai.com/docs",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error",
      "security_vulnerability",
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "output-validation",
      "validator-retry-loop",
      "rail-spec",
      "validator-hub"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Open-source (Apache-2.0). Validator Hub pulls third-party validator packages at runtime — treat hub installs as supply-chain surface and pin to audited versions. Some validators call external APIs (e.g., toxicity classifiers) and exfiltrate model output to those endpoints; review each validator's network posture before enabling.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 6718,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Canonical detection × post_generation × fully_autonomous exemplar for validator-retry on LLM output. Widely cited reference implementation of the \"validate-then-reask\" pattern, distinct from in-generation constrained decoding (Outlines/Guidance).\n"
  },
  {
    "id": "guidance",
    "name": "Guidance",
    "tagline": "Programmable interleaving of prompt, control flow, and constrained generation.",
    "description": "Python library for programming LLMs with interleaved text, control flow, regex/CFG constraints, and tool calls in a single program. Constraints (select, regex, grammar, JSON) are enforced at decoding time via logit masking against open-weights backends, and via token-healing to avoid tokenizer-boundary artifacts. Supports Transformers, llama.cpp, and remote providers where feasible.\n",
    "first_released": "2023-04-15",
    "last_updated": "2026-01-20",
    "maturity_status": "stable",
    "published_by": "guidance-ai",
    "repository_url": "https://github.com/guidance-ai/guidance",
    "documentation_url": "https://github.com/guidance-ai/guidance",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "in_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error"
    ],
    "locus_of_control": [
      "model",
      "prompt"
    ],
    "implements_techniques": [
      "constrained-decoding",
      "grammar-guided-generation",
      "token-healing",
      "interleaved-prompt-program"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Open-source (MIT), originally from Microsoft Research, now under the guidance-ai org. Requires logit access for full constraint enforcement, so closed-API backends degrade to weaker modes. Programs execute local Python; treat third-party Guidance programs as untrusted code.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 21402,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies prevention × in_generation × fully_autonomous alongside Outlines, but with a program-first ergonomics layer (interleaved prompting + constraints) rather than pure FSM compilation. Included to mark that the in-generation cell has multiple distinct approaches.\n"
  },
  {
    "id": "helicone",
    "name": "Helicone",
    "tagline": "Open-source LLM observability proxy with one-line integration and caching.",
    "description": "Open-source LLM observability platform that sits as a proxy in front of provider APIs. Integration is a one-line base-URL swap, after which every request and response is logged, cost-accounted, and made available for dataset creation, prompt experimentation, and custom evaluation. Also offers async logging via SDK for users who prefer not to proxy. Self-hostable core plus a managed cloud tier sharing the same data model.\n",
    "first_released": "2023-07-07",
    "last_updated": "2026-04-05",
    "maturity_status": "stable",
    "published_by": "helicone",
    "repository_url": "https://github.com/Helicone/helicone",
    "documentation_url": "https://docs.helicone.ai",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "logic_error",
      "context_pollution"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "proxy-logging",
      "async-sdk-logging",
      "cost-accounting",
      "dataset-capture",
      "prompt-experimentation"
    ],
    "integration_surfaces": [
      "proxy_gateway",
      "http_service",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Core project is open-source (Apache-2.0) and self-hostable, which avoids egressing prompts to a third party for IP-sensitive workloads. In proxy mode, API keys and full request bodies transit Helicone — self-host or use async SDK mode for high-sensitivity deployments. Standard redaction hooks apply; audit what payload fields are captured before enabling on production traffic.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 5541,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies detection × post_generation × fully_autonomous with a proxy-first architecture that distinguishes it from SDK-first peers (Langfuse, LangSmith, Phoenix). The proxy posture is load-bearing because it captures traffic with zero instrumentation, which is a materially different supervision surface even though the facet cell is shared.\n"
  },
  {
    "id": "helm",
    "name": "helm",
    "tagline": "Holistic Evaluation of Language Models (HELM).",
    "description": "Stanford CRFM's HELM is a benchmark suite covering accuracy, calibration, robustness, fairness, bias, toxicity, and efficiency across a scenario × metric matrix. Establishes the 'holistic evaluation' template that many later LLM evaluation efforts cite.\n",
    "first_released": "2021-11-29",
    "last_updated": "2026-04-23",
    "maturity_status": "experimental",
    "published_by": "stanford-crfm",
    "repository_url": "https://github.com/stanford-crfm/helm",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error",
      "scope_creep",
      "security_vulnerability"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "scenario-metric-matrix",
      "holistic-eval",
      "robustness-scenarios"
    ],
    "integration_surfaces": [
      "cli",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 2758,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Canonical holistic-eval suite — represents the research branch of LLM evaluation that keeps safety / bias / robustness as first-class metrics alongside accuracy.\n"
  },
  {
    "id": "instructor",
    "name": "Instructor",
    "tagline": "Pydantic-typed structured output for LLM APIs with automatic retries.",
    "description": "Thin Python library that patches LLM clients (OpenAI, Anthropic, Gemini, Cohere, Mistral, etc.) to return Pydantic models. Requests a tool/function-call response shape, parses, validates against the Pydantic schema, and retries with the validation error fed back as context on failure. Post-generation validation, not constrained decoding — relies on the provider's own JSON/tool-call mode for the initial shape.\n",
    "first_released": "2023-08-01",
    "last_updated": "2026-03-25",
    "maturity_status": "stable",
    "published_by": "jxnl",
    "repository_url": "https://github.com/jxnl/instructor",
    "documentation_url": "https://python.useinstructor.com",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "pydantic-schema-validation",
      "validator-retry-loop",
      "function-calling-coercion"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Open-source (MIT). Ships as a thin client patch — supply-chain surface is small but the library is broadly installed, so pin versions. Does not itself transmit data anywhere; all traffic goes through the underlying provider SDK whose posture applies.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 12816,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Detection × post_generation × fully_autonomous exemplar for the \"Pydantic schema + reask on ValidationError\" pattern. Useful contrast to PydanticAI (full agent framework) and to Outlines (in-generation constraints): Instructor is the minimal library expression of the validate-and-retry paradigm.\n"
  },
  {
    "id": "isort",
    "name": "isort",
    "tagline": "Sorts and groups Python imports automatically.",
    "description": "isort rewrites Python import statements into a canonical order (stdlib, third-party, first-party, local) with consistent grouping and spacing. By making import layout deterministic it eliminates a common source of cosmetic diffs and prevents reviewers from chasing style nits — a small but real check on scope creep in PR review. Configuration lives in `pyproject.toml [tool.isort]`, which the audit detector now recognises.\n",
    "first_released": "2013-09-15",
    "last_updated": "2026-04-15",
    "maturity_status": "stable",
    "published_by": "timothycrosley",
    "repository_url": "https://github.com/PyCQA/isort",
    "documentation_url": "https://pycqa.github.io/isort/",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "import-sorting",
      "ast-rewriting",
      "deterministic-style"
    ],
    "integration_surfaces": [
      "cli",
      "ci_app",
      "ide_plugin"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [
      "python"
    ],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Standard dev-tool security posture; no special considerations. isort runs locally and rewrites import blocks based on AST inspection only.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 6000,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "editorial_notes": "Added 2026-05-01 to fill the pyproject `[tool.*]` detection path so the audit no longer reports isort as an unknown tool.\n",
    "inclusion_rationale": "Occupies prevention × post_generation × fully_autonomous for scope_creep in the Python lane, alongside Black on a narrower formatting surface. Including it lets the audit detector resolve `[tool.isort]` blocks in legacy configs that have not yet migrated to Ruff's import sorter.\n"
  },
  {
    "id": "judgeval",
    "name": "judgeval",
    "tagline": "The open source post-building layer for agents: evals, traces, and regressions.",
    "description": "Judgeval is an open-source 'post-building' evaluation layer for agents from Judgment Labs. Pairs trace capture with regression testing, environment-as-data, and LLM-as-judge graders. Designed to be added after an agent is built to monitor quality and catch regressions in CI.\n",
    "first_released": "2024-10-25",
    "last_updated": "2026-04-21",
    "maturity_status": "experimental",
    "published_by": "judgment-labs",
    "repository_url": "https://github.com/JudgmentLabs/judgeval",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "scope_creep",
      "logic_error",
      "test_manipulation"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "agent-regression-tests",
      "llm-as-judge",
      "environment-as-data"
    ],
    "integration_surfaces": [
      "library",
      "cli"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 1024,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Fills the 'CI regression harness for agents' niche. Distinct from promptfoo (prompt-level regression) by operating on full agent runs and environment data.\n"
  },
  {
    "id": "langfuse",
    "name": "Langfuse",
    "tagline": "Open-source LLM engineering platform for tracing, evals, and prompt management.",
    "description": "Open-source observability and evaluation platform for LLM applications. Captures nested traces (trace → span → generation), stores prompts with versioning, runs dataset-driven and LLM-as-judge evaluations, and exposes a scoring API for human and programmatic feedback. Framework-agnostic Python/JS SDKs plus integrations with LangChain, LlamaIndex, and OpenTelemetry. Self-hostable via Docker compose or a managed cloud offering.\n",
    "first_released": "2023-07-01",
    "last_updated": "2026-04-10",
    "maturity_status": "stable",
    "published_by": "langfuse",
    "repository_url": "https://github.com/langfuse/langfuse",
    "documentation_url": "https://langfuse.com/docs",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "logic_error",
      "context_pollution",
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "trace-capture",
      "llm-as-judge-eval",
      "dataset-regression-eval",
      "prompt-versioning",
      "scoring-api",
      "opentelemetry-integration"
    ],
    "integration_surfaces": [
      "http_service",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Core project is open-source (MIT) and self-hostable, which makes it the canonical data-residency-friendly alternative to hosted observability. The managed cloud offering shares the same data model; audit retention and regional-hosting posture before routing production traffic. Prompts and completions are the primary payload — redact secrets and PII via the SDK hooks before capture.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 25839,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies detection × post_generation × fully_autonomous, paralleling LangSmith but with an open-source self-hostable core. Worth modeling as a distinct node because the license and deployment posture change the threat model materially for regulated or IP-sensitive deployments.\n"
  },
  {
    "id": "langgraph",
    "name": "LangGraph",
    "tagline": "Graph-structured state machines for stateful multi-agent orchestration.",
    "description": "Open-source orchestration library from LangChain for building stateful agent workflows as directed graphs with typed shared state, explicit nodes, and conditional edges. Supervision primitives: graph-state schema enforcement that scopes what each node can read/write, checkpointing for time-travel and replay, interrupt points for human-in-the-loop approval, and persistence for durable long-running runs. Positions itself as the disciplined alternative to unstructured agent loops.\n",
    "first_released": "2024-01-08",
    "last_updated": "2026-04-10",
    "maturity_status": "stable",
    "published_by": "langchain-ai",
    "repository_url": "https://github.com/langchain-ai/langgraph",
    "documentation_url": "https://langchain-ai.github.io/langgraph/",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "pre_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "scope_creep",
      "context_pollution"
    ],
    "locus_of_control": [
      "prompt",
      "context"
    ],
    "implements_techniques": [
      "graph-state-machine",
      "typed-shared-state",
      "checkpointing",
      "interrupt-points",
      "conditional-edges"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Open-source (MIT). Library code runs in-process with agent credentials; no hosted dependency required for core use, though LangGraph Cloud and LangSmith integration are opt-in hosted surfaces. Audit node implementations for tool-call and exfiltration surface — graph structure constrains control flow but not what individual nodes do.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 30108,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies prevention × pre_generation × fully_autonomous for the \"structured agent orchestration\" archetype. Graph-state schemas are a pre-generation contract: each node sees only declared state slices, which bounds context pollution and scope creep before generation rather than filtering outputs after.\n"
  },
  {
    "id": "langsmith",
    "name": "LangSmith",
    "tagline": "Hosted LLM observability, tracing, and evaluation platform.",
    "description": "LangChain's commercial observability platform for LLM and agent applications. Captures structured traces of chains, tool calls, and agent trajectories; supports dataset-driven offline evaluation with custom and LLM-as-judge evaluators; provides prompt versioning and a feedback API for production quality tracking. Runs against any framework via the tracing SDK, not only LangChain. Supervision locus is post-hoc: traces and evals surface regressions rather than preventing them at emission.\n",
    "first_released": "2023-07-18",
    "last_updated": "2026-04-10",
    "maturity_status": "stable",
    "published_by": "langchain-ai",
    "documentation_url": "https://docs.smith.langchain.com",
    "license": "proprietary",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "logic_error",
      "context_pollution",
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "trace-capture",
      "llm-as-judge-eval",
      "dataset-regression-eval",
      "prompt-versioning",
      "feedback-api"
    ],
    "integration_surfaces": [
      "http_service",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Proprietary hosted SaaS; prompts, completions, and intermediate payloads are sent to LangSmith's backend by default. A self-hosted deployment is available on enterprise plans for teams with data residency or IP-sensitivity constraints. Audit what payload fields are captured and consider redaction hooks before enabling on production traffic containing secrets or customer data.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "contributors": [],
    "inclusion_rationale": "Occupies detection × post_generation × fully_autonomous for the observability-and-evals archetype. Traces are the audit substrate for post-hoc detection of logic errors, drift, and scope creep in agent trajectories — a supervision role that is analytically distinct from output validators that act on a single call.\n"
  },
  {
    "id": "letta",
    "name": "letta",
    "tagline": "Stateful agents framework with memory, reasoning, and context management (formerly MemGPT).",
    "description": "Letta (formerly MemGPT) introduces 'MemGPT-style' long-term memory for LLM agents: tiered context buffers, summarization, and explicit memory tools that let agents write to and retrieve from long-term storage. Addresses runaway context pollution.\n",
    "first_released": "2023-10-11",
    "last_updated": "2026-04-12",
    "maturity_status": "stable",
    "published_by": "letta-ai",
    "repository_url": "https://github.com/letta-ai/letta",
    "license": "Apache-2.0",
    "control_paradigm": "prevention",
    "temporal_phase": "pre_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "context_pollution",
      "scope_creep"
    ],
    "locus_of_control": [
      "context",
      "environment"
    ],
    "implements_techniques": [
      "tiered-memory",
      "summarization",
      "memory-tools"
    ],
    "integration_surfaces": [
      "library",
      "http_service"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 22238,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Memory/context-management cell — supervision as bounded-context discipline rather than runtime gating.\n"
  },
  {
    "id": "lighteval",
    "name": "lighteval",
    "tagline": "Hugging Face all-in-one toolkit for evaluating LLMs across multiple backends.",
    "description": "Hugging Face's lightweight LLM evaluation toolkit. Supports multiple inference backends (vLLM, Accelerate, nanotron) and is the backbone of the Hugging Face Open LLM Leaderboard v2. Positioned as a leaner, HF-native successor to lm-evaluation-harness.\n",
    "first_released": "2024-01-26",
    "last_updated": "2026-04-17",
    "maturity_status": "experimental",
    "published_by": "huggingface",
    "repository_url": "https://github.com/huggingface/lighteval",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "multi-backend-eval",
      "open-llm-leaderboard-v2"
    ],
    "integration_surfaces": [
      "cli",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 2387,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "HF-native leaderboard harness. Differentiated from lm-evaluation-harness by tighter vLLM/Accelerate integration and faster eval runs.\n"
  },
  {
    "id": "litellm",
    "name": "litellm",
    "tagline": "Call any LLM using the OpenAI API format — proxy server and Python SDK.",
    "description": "LiteLLM is an LLM proxy + Python SDK that exposes 100+ model providers behind the OpenAI API. Functions as an enterprise gateway with virtual keys, spend tracking, failover, and rate limits. De-facto standard for provider-agnostic LLM access.\n",
    "first_released": "2023-07-27",
    "last_updated": "2026-04-23",
    "maturity_status": "stable",
    "published_by": "berri-ai",
    "repository_url": "https://github.com/BerriAI/litellm",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "in_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "scope_creep",
      "security_vulnerability"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "provider-shim",
      "virtual-keys",
      "spend-tracking",
      "failover"
    ],
    "integration_surfaces": [
      "library",
      "proxy_gateway"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 44383,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Multi-provider gateway cell — canonical point of comparison for bifrost and manifest.\n"
  },
  {
    "id": "llm-guard",
    "name": "llm-guard",
    "tagline": "The Security Toolkit for LLM Interactions.",
    "description": "ProtectAI's LLM Guard is a runtime security layer for LLM interactions. Pluggable input/output scanners (PII, secrets, prompt injection, jailbreak detection, profanity, toxicity, competitor detection) can be composed as a gate in front of LLM calls, with configurable allow/deny policies.\n",
    "first_released": "2023-07-27",
    "last_updated": "2025-12-15",
    "maturity_status": "experimental",
    "published_by": "protectai",
    "repository_url": "https://github.com/protectai/llm-guard",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "in_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "security_vulnerability",
      "fabrication",
      "scope_creep",
      "context_pollution"
    ],
    "locus_of_control": [
      "prompt",
      "environment"
    ],
    "implements_techniques": [
      "input-scanners",
      "output-scanners",
      "pii-detection",
      "prompt-injection-detection"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 2855,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Canonical runtime-guard toolkit: covers both directions (user→LLM and LLM→user). Differentiated from rebuff by a wider scanner catalogue and from nemo-guardrails by being library-first (no DSL).\n"
  },
  {
    "id": "lm-evaluation-harness",
    "name": "lm-evaluation-harness",
    "tagline": "A framework for few-shot evaluation of language models.",
    "description": "EleutherAI's canonical few-shot LLM evaluation framework. Implements 200+ standard benchmarks (MMLU, HellaSwag, ARC, TruthfulQA, HumanEval, GSM8K, BBH) in a unified harness. Backbone of Hugging Face's Open LLM Leaderboard.\n",
    "first_released": "2020-08-28",
    "last_updated": "2026-04-22",
    "maturity_status": "stable",
    "published_by": "eleutherai",
    "repository_url": "https://github.com/EleutherAI/lm-evaluation-harness",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "few-shot-eval",
      "standard-benchmarks",
      "reference-harness"
    ],
    "integration_surfaces": [
      "cli",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 12298,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Foundational LLM eval harness: cited as 'the harness' in leaderboard and model-release contexts. Differentiated from lighteval by a larger task catalog and older, broader adoption.\n"
  },
  {
    "id": "lmms-eval",
    "name": "lmms-eval",
    "tagline": "One-for-all multimodal evaluation toolkit across text, image, video, and audio tasks.",
    "description": "LMMs-Eval is the EvolvingLMMs-Lab's multimodal evaluation harness — a multimodal counterpart to lm-evaluation-harness. Implements 100+ benchmarks across text, image, video, and audio tasks in a consistent interface.\n",
    "first_released": "2024-03-07",
    "last_updated": "2026-04-23",
    "maturity_status": "experimental",
    "published_by": "evolvinglmms-lab",
    "repository_url": "https://github.com/EvolvingLMMs-Lab/lmms-eval",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "multimodal-benchmarks",
      "unified-harness"
    ],
    "integration_surfaces": [
      "cli",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 4052,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Fills the multimodal-eval cell. Differentiated from lm-evaluation-harness (text-only) and lighteval by its native multimodal scope.\n"
  },
  {
    "id": "lmnr",
    "name": "lmnr",
    "tagline": "Laminar — open-source observability platform purpose-built for AI agents.",
    "description": "Laminar is an open-source observability platform purpose-built for agents. Captures long-running agent trajectories, browser sessions, and tool calls, and pairs trace capture with online evals and prompt/tool versioning. YC S24 batch.\n",
    "first_released": "2024-08-29",
    "last_updated": "2026-04-22",
    "maturity_status": "experimental",
    "published_by": "lmnr-ai",
    "repository_url": "https://github.com/lmnr-ai/lmnr",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error",
      "scope_creep",
      "incomplete_execution",
      "cascading_failure"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "agent-trajectory-capture",
      "browser-session-replay",
      "online-evals"
    ],
    "integration_surfaces": [
      "http_service",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 2798,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Purpose-built for the agent-trajectory + browser-session case rather than generic LLM call traces. Useful point in the observability cell where the unit of observation is the agent run, not a single LLM invocation.\n"
  },
  {
    "id": "logfire",
    "name": "logfire",
    "tagline": "Pydantic Logfire — observability for AI agents, LLMs, and Python apps built on OpenTelemetry.",
    "description": "Pydantic Logfire is a Python-first observability platform from the Pydantic team, built on OpenTelemetry. First-class integration with Pydantic AI, LangChain, OpenAI, and FastAPI, making it a natural fit for Python-centric agent stacks. Captures LLM prompts/completions, tool calls, and general application traces.\n",
    "first_released": "2024-04-23",
    "last_updated": "2026-04-23",
    "maturity_status": "experimental",
    "published_by": "pydantic",
    "repository_url": "https://github.com/pydantic/logfire",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error",
      "scope_creep",
      "context_pollution"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "opentelemetry-tracing",
      "llm-instrumentation",
      "pydantic-ai-integration"
    ],
    "integration_surfaces": [
      "http_service",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 4191,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Represents the Python-native + Pydantic-ecosystem observability path. Complements pydantic-ai as the canonical Pydantic-stack tracing backend.\n"
  },
  {
    "id": "magic-mcp",
    "name": "magic-mcp",
    "tagline": "Magic MCP — generate frontend components inside Cursor, Windsurf, and Cline.",
    "description": "21st.dev's Magic MCP is an MCP server that generates and updates frontend components from natural language inside coding agents (Cursor, Windsurf, Cline). Provides a curated design system + component catalog, constraining generation to a known-good template set.\n",
    "first_released": "2025-02-19",
    "last_updated": "2026-02-17",
    "maturity_status": "experimental",
    "published_by": "21st-dev",
    "repository_url": "https://github.com/21st-dev/magic-mcp",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "post_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "scope_creep",
      "fabrication",
      "obsolescence"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "component-generation",
      "mcp-server",
      "template-catalog"
    ],
    "integration_surfaces": [
      "mcp_server"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 4776,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "MCP-mediated codegen cell: distinct from generic code-completion by hosting a curated template catalog.\n"
  },
  {
    "id": "manifest",
    "name": "manifest",
    "tagline": "Smart model routing for personal AI agents.",
    "description": "Manifest is an open-source model-routing gateway for AI agents. Routes each request to the cheapest model that can handle it and falls back across providers on failure. Positions itself as a cost-control layer for personal/indie agent deployments.\n",
    "first_released": "2022-09-27",
    "last_updated": "2026-04-23",
    "maturity_status": "stable",
    "published_by": "manifest",
    "repository_url": "https://github.com/mnfst/manifest",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "in_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "scope_creep",
      "context_pollution",
      "cascading_failure"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "smart-routing",
      "cost-optimization",
      "provider-failover"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 5555,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Cost-routing branch of the gateway cell: complements enterprise gateways (bifrost) with a lighter, dev-first UX.\n"
  },
  {
    "id": "mastra",
    "name": "mastra",
    "tagline": "The TypeScript AI agent framework — assistants, RAG, observability; supports any LLM.",
    "description": "Mastra is a TypeScript agent framework with first-class support for workflows, RAG, evals, and observability. Positions itself as a lightweight alternative to LangChain-js for TS/Node ecosystems.\n",
    "first_released": "2024-08-06",
    "last_updated": "2026-04-23",
    "maturity_status": "stable",
    "published_by": "mastra-ai",
    "repository_url": "https://github.com/mastra-ai/mastra",
    "license": "Elastic-2.0",
    "control_paradigm": "prevention",
    "temporal_phase": "pre_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "scope_creep",
      "context_pollution"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "ts-agents",
      "workflow-authoring",
      "rag-integration",
      "builtin-evals"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 23247,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "TypeScript-agent-framework cell. Complements voltagent (runtime-observability focus) by centring workflow authoring + RAG integration.\n"
  },
  {
    "id": "mcp-playwright",
    "name": "mcp-playwright",
    "tagline": "Community Playwright MCP server for Claude Desktop, Cursor, Cline, and others.",
    "description": "ExecuteAutomation's community Playwright MCP wraps Playwright as an MCP server compatible with Claude Desktop, Cursor, Cline, and similar LLM clients. Differs from microsoft/playwright-mcp by supporting broader action surface (API testing) and by being community-maintained.\n",
    "first_released": "2024-12-03",
    "last_updated": "2025-12-13",
    "maturity_status": "stable",
    "published_by": "execute-automation",
    "repository_url": "https://github.com/executeautomation/mcp-playwright",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "scope_creep",
      "security_vulnerability"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "mcp-server",
      "playwright-wrapper",
      "api-testing"
    ],
    "integration_surfaces": [
      "mcp_server"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 5461,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Community counterpart to Microsoft's official Playwright MCP — diverges on feature set and release cadence.\n"
  },
  {
    "id": "mcp-server-browserbase",
    "name": "mcp-server-browserbase",
    "tagline": "MCP server to control a browser via Browserbase and Stagehand.",
    "description": "Browserbase's official MCP server exposes Browserbase-hosted headless browsers to MCP clients. Pairs with Stagehand (typed browser-agent SDK) to provide a managed sandboxed browser backend for agent computer-use.\n",
    "first_released": "2024-12-05",
    "last_updated": "2026-03-31",
    "maturity_status": "experimental",
    "published_by": "browserbase",
    "repository_url": "https://github.com/browserbase/mcp-server-browserbase",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "scope_creep",
      "security_vulnerability"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "mcp-server",
      "browserbase-backend",
      "stagehand-integration"
    ],
    "integration_surfaces": [
      "mcp_server",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 3280,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Cloud-sandbox-browser MCP cell — distinct from playwright-mcp (local headless) and browser-mcp (user's live browser).\n"
  },
  {
    "id": "mcp-toolbox",
    "name": "genai-toolbox",
    "tagline": "MCP Toolbox for Databases — open-source MCP server for databases.",
    "description": "Google's genai-toolbox (MCP Toolbox) is an open-source MCP server that safely exposes database access to LLM agents. Provides per-tool allow-lists, schema-aware query generation, and auditable execution logs — turning direct DB access into a supervised interface.\n",
    "first_released": "2024-06-07",
    "last_updated": "2026-04-23",
    "maturity_status": "stable",
    "published_by": "google",
    "repository_url": "https://github.com/googleapis/genai-toolbox",
    "license": "Apache-2.0",
    "control_paradigm": "prevention",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "security_vulnerability",
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "mcp-db-server",
      "schema-aware-sql",
      "audit-log"
    ],
    "integration_surfaces": [
      "mcp_server"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 14769,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Database-access MCP cell. Differentiated from generic SQL MCPs by schema-aware generation and an opinionated safety posture.\n"
  },
  {
    "id": "mlflow",
    "name": "mlflow",
    "tagline": "Open-source platform for the machine learning lifecycle — experiments, tracking, models, and LLM evaluation.",
    "description": "MLflow is the open-source ML lifecycle platform stewarded by Databricks. Recent LLM-focused additions (MLflow 2.x) bring LLM tracing (OpenTelemetry), LLM evaluation, and a deployments registry — making it usable as an observability + eval backbone for LLM applications alongside classic ML.\n",
    "first_released": "2018-06-05",
    "last_updated": "2026-04-23",
    "maturity_status": "stable",
    "published_by": "databricks",
    "repository_url": "https://github.com/mlflow/mlflow",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error",
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "llm-tracing",
      "experiments",
      "model-registry",
      "llm-evaluation"
    ],
    "integration_surfaces": [
      "http_service",
      "library",
      "cli"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 25505,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Represents the 'classical-MLOps-extended-to-LLMs' point: teams with MLflow infrastructure can adopt LLM evaluation without a greenfield tool. Differentiated from langfuse/phoenix by its ML lifecycle lineage and Databricks ecosystem integration.\n"
  },
  {
    "id": "modal",
    "name": "Modal",
    "tagline": "Serverless sandboxed runtime with gVisor-isolated containers for agent code execution.",
    "description": "Serverless compute platform whose Sandbox API exposes per-call, gVisor- isolated containers designed for executing AI-generated code. Supervision primitives: ephemeral container lifecycle, fine-grained CPU/GPU/memory limits, filesystem and network policy controls, and snapshot/restore for state checkpointing. Tuned for fully_autonomous agent back-ends that need to run untrusted code at low startup cost.\n",
    "first_released": "2022-10-01",
    "last_updated": "2026-04-12",
    "maturity_status": "stable",
    "published_by": "modal-labs",
    "documentation_url": "https://modal.com/docs/guide/sandbox",
    "license": "proprietary",
    "control_paradigm": "recovery",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "supply_chain_attack",
      "security_vulnerability",
      "logic_error"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "sandboxed-execution",
      "gvisor-isolation",
      "resource-limiting",
      "snapshot-restore"
    ],
    "integration_surfaces": [
      "library",
      "cli"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Proprietary hosted platform. Sandboxes run on gVisor for kernel-surface reduction; still evaluate tenancy, data-residency, and egress-control guarantees for sensitive workloads. Agent-generated code should be treated as untrusted — pin network policy and secret scopes explicitly per sandbox.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "contributors": [],
    "inclusion_rationale": "Occupies recovery × post_generation × fully_autonomous alongside E2B and Daytona. Included because gVisor-based isolation and snapshot/restore represent a distinct point in the sandbox-provider design space relevant to autonomous coding-agent deployments.\n"
  },
  {
    "id": "mypy",
    "name": "mypy",
    "tagline": "Static type checker for Python. Catches mismatched types before runtime.",
    "description": "mypy is the original gradual-typing checker for Python, reading PEP 484 type hints across a project and reporting type-incompatibility, missing-attribute, and contract-mismatch errors before code runs. It guards against logic errors caused by silent type drift and against fabricated APIs (calls to attributes that do not exist). Configuration lives in `pyproject.toml [tool.mypy]`, which the audit detector now recognises.\n",
    "first_released": "2012-04-01",
    "last_updated": "2026-04-15",
    "maturity_status": "stable",
    "published_by": "python",
    "repository_url": "https://github.com/python/mypy",
    "documentation_url": "https://mypy.readthedocs.io/",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "logic_error",
      "fabrication"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "static-type-checking",
      "gradual-typing",
      "type-inference",
      "stub-resolution"
    ],
    "integration_surfaces": [
      "cli",
      "ci_app",
      "ide_plugin"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [
      "python"
    ],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Standard dev-tool security posture; no special considerations. mypy runs locally and does not execute target code, only parses and walks its AST.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 18000,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "editorial_notes": "Added 2026-05-01 to fill the pyproject `[tool.*]` detection path so the audit no longer reports mypy as an unknown tool.\n",
    "inclusion_rationale": "Occupies detection × post_generation × fully_autonomous for logic_error and fabrication in the Python lane. The reference implementation of PEP 484; missing it leaves the audit detector unable to recognise the most widely-deployed Python type-checking gate.\n"
  },
  {
    "id": "nemo-guardrails",
    "name": "NeMo Guardrails",
    "tagline": "Colang-based programmable guardrails for LLM applications.",
    "description": "NVIDIA's open-source toolkit for adding programmable rails around LLM applications. Uses Colang, a DSL for specifying conversation flows, topical rails, safety rails, fact-checking rails, and jailbreak detection. Rails execute around the LLM call as input checks, dialog flow control, and output checks. Integrates with LangChain and supports pluggable policy models.\n",
    "first_released": "2023-04-25",
    "last_updated": "2026-02-10",
    "maturity_status": "stable",
    "published_by": "nvidia",
    "repository_url": "https://github.com/NVIDIA/NeMo-Guardrails",
    "documentation_url": "https://docs.nvidia.com/nemo/guardrails",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "scope_creep",
      "security_vulnerability",
      "context_pollution",
      "fabrication"
    ],
    "locus_of_control": [
      "prompt",
      "environment"
    ],
    "implements_techniques": [
      "colang-policy-dsl",
      "topical-rails",
      "fact-checking-rail",
      "jailbreak-detection",
      "output-moderation"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Open-source (Apache-2.0) from NVIDIA. Colang rails can call external moderation/classifier models — audit those endpoints for data-residency. Rails themselves are code; treat rail bundles pulled from third parties as supply-chain surface. NVIDIA vendor-maintained; tracks NeMo stack versions.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 6028,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies detection × post_generation × fully_autonomous with a policy-language approach distinct from Pydantic-schema validators. Acts primarily as output/flow moderation, not constrained decoding — hence post_generation rather than in_generation.\n"
  },
  {
    "id": "nono",
    "name": "nono",
    "tagline": "Capability-based sandbox for securely running AI agents with zero setup",
    "description": "nono is a capability-based, multiplexing sandbox tool designed for developers to securely run AI agents. It provides zero-setup security isolation enabling a seamless path from development to production without additional infrastructure or latency overhead.",
    "first_released": "2026-01-31",
    "last_updated": "2026-04-23",
    "maturity_status": "experimental",
    "repository_url": "https://github.com/always-further/nono",
    "license": "Apache-2.0",
    "control_paradigm": "prevention",
    "temporal_phase": "in_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "security_vulnerability",
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [],
    "integration_surfaces": [
      "cli",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 2085,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies prevention × in_generation × fully_autonomous as a capability-based sandbox that mediates an agent's syscalls while it is still acting, rather than isolating a whole job at the container boundary (contrast modal, e2b, daytona, which sit in recovery × post_generation). The novel primitive is zero-setup multiplexed capability enforcement so a single host can safely run many concurrent agent sessions without per-agent VM provisioning, useful as the local-dev-mode exemplar of in-generation environment control.\n"
  },
  {
    "id": "openai-evals",
    "name": "evals",
    "tagline": "Framework for evaluating LLMs and LLM systems, plus an open registry of benchmarks.",
    "description": "OpenAI Evals is a framework for evaluating LLMs and LLM systems and a community registry of benchmarks. Supports both traditional reference graders and model-graded evals. Distinct from lm-evaluation-harness by emphasizing user-contributed evals over a fixed benchmark suite.\n",
    "first_released": "2023-01-23",
    "last_updated": "2026-04-14",
    "maturity_status": "stable",
    "published_by": "openai",
    "repository_url": "https://github.com/openai/evals",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error",
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "model-graded-evals",
      "eval-registry",
      "reference-graders"
    ],
    "integration_surfaces": [
      "library",
      "cli"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 18262,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Canonical 'bring-your-own-eval' registry: central artifact for OpenAI-model comparisons.\n"
  },
  {
    "id": "openlit",
    "name": "openlit",
    "tagline": "Open-source AI engineering platform: OpenTelemetry-native LLM observability, guardrails, evaluations, prompt management.",
    "description": "OpenLIT is a single open-source AI engineering platform that layers OpenTelemetry-native LLM observability, guardrails, evaluations, and prompt management on top of existing OTel collectors. Designed for teams that already have OTel infrastructure and want to extend it to LLM workloads without a separate control plane.\n",
    "first_released": "2024-01-23",
    "last_updated": "2026-04-20",
    "maturity_status": "experimental",
    "published_by": "openlit",
    "repository_url": "https://github.com/openlit/openlit",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "scope_creep",
      "context_pollution"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "opentelemetry",
      "guardrails",
      "evals",
      "prompt-management"
    ],
    "integration_surfaces": [
      "http_service",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 2388,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies the 'OTel-collector-native' point in observability — distinct from langfuse/phoenix by being designed to plug into existing OpenTelemetry backends rather than operate a standalone control plane.\n"
  },
  {
    "id": "opik",
    "name": "opik",
    "tagline": "Debug, evaluate, and monitor LLM applications, RAG systems, and agentic workflows.",
    "description": "Opik is Comet ML's open-source platform for debugging, evaluating, and monitoring LLM applications, RAG pipelines, and multi-agent workflows. Provides trace capture, LLM-as-judge graders, test dataset management, and production monitoring dashboards. Self-hostable alongside a SaaS offering.\n",
    "first_released": "2023-05-10",
    "last_updated": "2026-04-23",
    "maturity_status": "stable",
    "published_by": "comet-ml",
    "repository_url": "https://github.com/comet-ml/opik",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error",
      "scope_creep",
      "incomplete_execution"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "llm-tracing",
      "llm-as-judge",
      "test-datasets",
      "production-monitoring"
    ],
    "integration_surfaces": [
      "http_service",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 18988,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies detection × post_generation × full_hitl. Differentiated from langfuse by being Comet-ecosystem-integrated (shared ML experiment-tracking backbone) and from phoenix by its agent-focused trace model.\n"
  },
  {
    "id": "outlines",
    "name": "Outlines",
    "tagline": "Constrained decoding via finite-state machines over regex, JSON Schema, and CFGs.",
    "description": "Python library for structured generation that compiles regex, JSON Schema, or a context-free grammar into a finite-state machine and masks the LLM's logits at every decoding step so only tokens leading to valid output can be sampled. Output is guaranteed to match the schema by construction — no retry loop required. Backends include Transformers, vLLM, llama.cpp, MLX, and a hosted service (.txt).\n",
    "first_released": "2023-06-01",
    "last_updated": "2026-03-01",
    "maturity_status": "stable",
    "published_by": "dottxt-ai",
    "repository_url": "https://github.com/dottxt-ai/outlines",
    "documentation_url": "https://dottxt-ai.github.io/outlines",
    "license": "Apache-2.0",
    "control_paradigm": "prevention",
    "temporal_phase": "in_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error"
    ],
    "locus_of_control": [
      "model"
    ],
    "implements_techniques": [
      "constrained-decoding",
      "logit-masking",
      "fsm-guided-generation",
      "json-schema-decoding",
      "cfg-guided-generation"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Open-source (Apache-2.0). Requires access to model logits, so it works with open-weights backends (Transformers/vLLM/llama.cpp) but not with closed APIs that don't expose logit bias. FSM compilation happens locally; no data leaves the process. Hosted .txt service is a separate SaaS with its own data-handling posture.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 13718,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Canonical prevention × in_generation × fully_autonomous exemplar: logit-masking during decoding prevents schema-invalid tokens from ever being emitted. Distinct from post-generation validators — output is correct-by-construction rather than checked-after-the-fact.\n"
  },
  {
    "id": "pathway-llm-app",
    "name": "llm-app",
    "tagline": "Ready-to-run cloud templates for RAG, AI pipelines, and enterprise search with live data.",
    "description": "Pathway's llm-app provides production-ready templates for RAG and AI pipelines over live data streams. Emphasizes streaming ingestion and incremental re-indexing, addressing the obsolescence failure mode where RAG answers drift as underlying data changes.\n",
    "first_released": "2023-07-19",
    "last_updated": "2026-01-07",
    "maturity_status": "stable",
    "published_by": "pathway",
    "repository_url": "https://github.com/pathwaycom/llm-app",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "context_pollution",
      "obsolescence"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "streaming-rag",
      "incremental-indexing",
      "pipeline-templates"
    ],
    "integration_surfaces": [
      "library",
      "http_service"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 59931,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Live-data RAG cell — addresses obsolescence/freshness in RAG pipelines by incremental indexing, a concern that most batch-RAG tools understate.\n"
  },
  {
    "id": "pezzo",
    "name": "pezzo",
    "tagline": "The cloud-native open-source LLMOps platform.",
    "description": "Pezzo is a cloud-native open-source LLMOps platform providing observability, prompt versioning, A/B testing, and cost controls. Positioned for small teams looking to self-host their LLM ops.\n",
    "first_released": "2023-04-22",
    "last_updated": "2026-03-31",
    "maturity_status": "experimental",
    "published_by": "pezzo-labs",
    "repository_url": "https://github.com/pezzolabs/pezzo",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "scope_creep",
      "context_pollution",
      "fabrication"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "observability",
      "prompt-versioning",
      "ab-testing",
      "cost-controls"
    ],
    "integration_surfaces": [
      "http_service",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 3227,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Small-team self-hosted LLMOps cell — distinct from langfuse by a stronger Kubernetes-native posture.\n"
  },
  {
    "id": "playwright-mcp",
    "name": "playwright-mcp",
    "tagline": "Playwright Model Context Protocol server from Microsoft.",
    "description": "Microsoft Playwright MCP is the official MCP server wrapping Playwright. Exposes a controlled set of browser actions to LLM agents (click, type, navigate, screenshot) through MCP, so agent access to the browser is channelled through an auditable tool interface.\n",
    "first_released": "2025-03-21",
    "last_updated": "2026-04-22",
    "maturity_status": "stable",
    "published_by": "microsoft",
    "repository_url": "https://github.com/microsoft/playwright-mcp",
    "license": "Apache-2.0",
    "control_paradigm": "prevention",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "scope_creep",
      "security_vulnerability"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "mcp-server",
      "playwright-wrapper",
      "browser-actions"
    ],
    "integration_surfaces": [
      "mcp_server"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 31275,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Environment-gate for browser-using agents: MCP-mediated access rather than direct CDP, enabling per-tool allow-lists + auditing.\n"
  },
  {
    "id": "pr-agent",
    "name": "pr-agent",
    "tagline": "AI-powered tool for automated pull-request analysis, feedback, and suggestions.",
    "description": "Qodo (formerly CodiumAI) pr-agent automates PR review using LLMs. Produces structured code review, security hints, and test suggestions gated on a maintainer who approves diffs. Canonical exemplar for human-in-the-loop code-review supervision.\n",
    "first_released": "2023-07-05",
    "last_updated": "2026-04-21",
    "maturity_status": "stable",
    "published_by": "qodo",
    "repository_url": "https://github.com/Codium-ai/pr-agent",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "logic_error",
      "security_vulnerability",
      "fabrication"
    ],
    "locus_of_control": [
      "environment",
      "human"
    ],
    "implements_techniques": [
      "pr-review",
      "security-hints",
      "test-suggestions",
      "maintainer-gate"
    ],
    "integration_surfaces": [
      "ci_app",
      "cli"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 10968,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Code-review supervisor cell — distinct from generic linters / static analyzers by reasoning over diff intent and leaving humans as the final authority.\n"
  },
  {
    "id": "pre-commit",
    "name": "pre-commit",
    "tagline": "Multi-language framework that runs configurable hooks before commits land — denies the commit on hook failure.",
    "description": "pre-commit is the canonical git-hook orchestrator: developers declare a `.pre-commit-config.yaml` listing hook repositories (semgrep, ruff, black, custom shell scripts, etc.), and the framework runs them on every `git commit`, blocking the commit if any hook returns non-zero. This makes pre-commit the *prevention* primitive: bad code never lands in the repository, so downstream supervisors (CI gates, code review) see less noise. Hooks compose — pre-commit itself is paradigm-agnostic; the failure modes it addresses depend on which hooks are installed. This entry credits pre-commit with the failure modes its standard hook ecosystem most reliably guards against.\n",
    "first_released": "2014-09-01",
    "last_updated": "2026-04-15",
    "maturity_status": "stable",
    "published_by": "pre-commit",
    "repository_url": "https://github.com/pre-commit/pre-commit",
    "documentation_url": "https://pre-commit.com/",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "post_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "scope_creep",
      "security_vulnerability",
      "dependency_blindness",
      "fabrication"
    ],
    "locus_of_control": [
      "environment",
      "human"
    ],
    "implements_techniques": [
      "git-hook-orchestration",
      "hook-version-pinning",
      "parallel-hook-execution",
      "configurable-stage-gating"
    ],
    "integration_surfaces": [
      "cli",
      "ci_app"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [
      "python",
      "javascript",
      "typescript",
      "shell",
      "yaml",
      "json"
    ],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Hooks pull from upstream repositories pinned by SHA in `.pre-commit-config.yaml`; supply-chain risk is bounded to whatever the developer pins. `--no-verify` allows bypass per-commit (humans can override; visible in `git log`). Default config does not run hooks against `.git/`-tracked secrets — pair with `detect-private-key` hook for that surface.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "contributors": [],
    "inclusion_rationale": "Occupies prevention × post_generation × graduated_hitl in the canonical 3-axis grid. Distinct from CI-gate tools (semgrep, snyk) because the block point is the local commit, not the central CI run — so feedback is faster and bad code never enters the shared history. Distinct from IDE-level prevention (Cursor rules, CLAUDE.md) because the gate is language-agnostic and works across whichever agent generated the code. Standardised supply chain (every hook is a pinned upstream repo) makes pre-commit the closest thing the agentic-coding stack has to a hook-framework \"kernel\".\n"
  },
  {
    "id": "prettier",
    "name": "Prettier",
    "tagline": "Opinionated code formatter for JS/TS/CSS/HTML. Ends style debates.",
    "description": "Prettier is an opinionated multi-language formatter covering JavaScript, TypeScript, JSX, CSS, HTML, JSON, Markdown, and more. By collapsing the design space of \"how should this look\" into a small fixed set of options, Prettier prevents scope creep in code review (style nits) and removes a whole class of agent-introduced cosmetic diffs. Configuration lives in `.prettierrc` or `package.json` `prettier`, which the audit detector now recognises.\n",
    "first_released": "2017-01-22",
    "last_updated": "2026-04-15",
    "maturity_status": "stable",
    "published_by": "prettier",
    "repository_url": "https://github.com/prettier/prettier",
    "documentation_url": "https://prettier.io/docs/en/",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "code-formatting",
      "ast-rewriting",
      "deterministic-style"
    ],
    "integration_surfaces": [
      "cli",
      "ci_app",
      "ide_plugin"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [
      "javascript",
      "typescript",
      "css",
      "html",
      "json",
      "markdown"
    ],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Standard dev-tool security posture; Prettier runs locally and rewrites source from its parsed AST without executing it. Plugin ecosystem is sizeable — pin plugin versions in `package.json`.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 50000,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "editorial_notes": "Added 2026-05-01 to fill the package.json detection path so the audit no longer reports prettier as an unknown tool.\n",
    "inclusion_rationale": "Occupies prevention × post_generation × fully_autonomous for scope_creep across the JS/TS/CSS/HTML lanes. Mature workhorse; including it lets the audit detector resolve `prettier` blocks in package.json and credit projects for having a deterministic-style gate in place.\n"
  },
  {
    "id": "promptbench",
    "name": "promptbench",
    "tagline": "Unified evaluation framework for large language models with adversarial perturbations.",
    "description": "Microsoft PromptBench is a framework to benchmark LLM robustness under adversarial prompt perturbations. Evaluates stability, robustness, and reliability with defined attack taxonomies and measures prompt sensitivity of outputs.\n",
    "first_released": "2023-06-13",
    "last_updated": "2026-02-20",
    "maturity_status": "experimental",
    "published_by": "microsoft",
    "repository_url": "https://github.com/microsoft/promptbench",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "security_vulnerability",
      "logic_error"
    ],
    "locus_of_control": [
      "prompt",
      "environment"
    ],
    "implements_techniques": [
      "adversarial-perturbations",
      "prompt-sensitivity-metrics"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 2801,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Robustness-under-perturbation cell of LLM evaluation — pairs well with garak/promptmap (security) and helm (holistic).\n"
  },
  {
    "id": "promptflow",
    "name": "promptflow",
    "tagline": "Build high-quality LLM apps — from prototyping, testing to production deployment and monitoring.",
    "description": "Microsoft Prompt flow is an SDK + VS Code extension + Azure service for building, evaluating, and deploying LLM apps. Graph-based authoring, batch eval with built-in metrics, and Azure ML integration.\n",
    "first_released": "2023-06-30",
    "last_updated": "2026-04-21",
    "maturity_status": "stable",
    "published_by": "microsoft",
    "repository_url": "https://github.com/microsoft/promptflow",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error",
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "graph-authoring",
      "batch-eval",
      "azure-ml-integration"
    ],
    "integration_surfaces": [
      "library",
      "ide_plugin"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 11106,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Azure-integrated LLM dev-to-prod platform cell. Distinguished from semantic-kernel by centring evaluation and batch runs rather than runtime orchestration.\n"
  },
  {
    "id": "promptfoo",
    "name": "promptfoo",
    "tagline": "Test prompts, agents, and RAGs — red-teaming, pentesting, and vulnerability scanning for LLMs.",
    "description": "Promptfoo is a CLI + library for testing, evaluating, and red-teaming LLM applications. Supports side-by-side prompt comparison, LLM-as-judge assertions, adversarial red-team generation, and CI integrations. Used by enterprises to gate LLM deployments on pre-prod evals.\n",
    "first_released": "2023-04-28",
    "last_updated": "2026-04-23",
    "maturity_status": "stable",
    "published_by": "promptfoo",
    "repository_url": "https://github.com/promptfoo/promptfoo",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "security_vulnerability",
      "logic_error",
      "scope_creep",
      "test_manipulation"
    ],
    "locus_of_control": [
      "prompt",
      "environment"
    ],
    "implements_techniques": [
      "prompt-eval-harness",
      "red-team-attacks",
      "llm-as-judge",
      "ci-gates"
    ],
    "integration_surfaces": [
      "cli",
      "library",
      "ci_app"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 20465,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies the CLI-driven prompt-eval + red-team cell. Distinct from deepeval (Pytest-native unit tests) by leaning into YAML configs and shift-left red-teaming.\n"
  },
  {
    "id": "promptmap",
    "name": "promptmap",
    "tagline": "Security scanner for custom LLM applications.",
    "description": "Promptmap is a CLI security scanner for custom LLM apps. Discovers system prompt leakage, prompt injection, and jailbreaks across user-provided endpoints. Designed for app-sec teams who want a quick adversarial sweep of their production LLM feature.\n",
    "first_released": "2023-07-15",
    "last_updated": "2025-12-01",
    "maturity_status": "experimental",
    "published_by": "utkusen",
    "repository_url": "https://github.com/utkusen/promptmap",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "security_vulnerability",
      "fabrication",
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "system-prompt-leak-probe",
      "prompt-injection-probe",
      "jailbreak-probe"
    ],
    "integration_surfaces": [
      "cli"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 1181,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Fills the 'nmap-for-LLM-apps' niche: fast, single-purpose external scanner — complements broader scanners like garak which target the model rather than the deployed app.\n"
  },
  {
    "id": "pydantic-ai",
    "name": "PydanticAI",
    "tagline": "Structured-output agent framework with Pydantic-typed tool and return schemas.",
    "description": "Python agent framework that enforces Pydantic schemas on tool inputs and model outputs, with automatic retries on validation failure. Provides a dependency-injection system for testability and observability hooks for integration with Logfire.\n",
    "first_released": "2024-10-01",
    "last_updated": "2026-03-15",
    "maturity_status": "stable",
    "published_by": "pydantic",
    "repository_url": "https://github.com/pydantic/pydantic-ai",
    "documentation_url": "https://ai.pydantic.dev",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error",
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "structured-output-validation",
      "validator-retry-loop"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [
      "context-hub"
    ],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Open-source, Pydantic-backed. Signed releases via PyPI. Pin to a known version to avoid supply-chain drift. Review plugin ecosystem for any custom validators you adopt.\n",
    "signed_manifest": true,
    "openssf_scorecard_score": null,
    "stars": 16567,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies the detection × post_generation × fully_autonomous cell with a concrete implementation of validator-retry. Well-known, widely-adopted reference point.\n"
  },
  {
    "id": "pyright",
    "name": "Pyright",
    "tagline": "Microsoft's fast static type checker for Python.",
    "description": "Pyright is Microsoft's TypeScript-implemented static type checker for Python, designed for low-latency incremental analysis of large codebases. It powers the Pylance language server in VS Code and runs from the CLI in CI, catching type incompatibilities and references to non-existent attributes (the fabrication failure mode) before code ships. Configuration lives in `pyproject.toml [tool.pyright]` or in a dedicated `pyrightconfig.json`.\n",
    "first_released": "2019-03-29",
    "last_updated": "2026-04-15",
    "maturity_status": "stable",
    "published_by": "microsoft",
    "repository_url": "https://github.com/microsoft/pyright",
    "documentation_url": "https://microsoft.github.io/pyright/",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "logic_error",
      "fabrication"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "static-type-checking",
      "incremental-analysis",
      "type-inference",
      "language-server-protocol"
    ],
    "integration_surfaces": [
      "cli",
      "ci_app",
      "ide_plugin"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [
      "python"
    ],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Standard dev-tool security posture; Pyright runs locally and analyses source statically without executing it. Released under the MIT license with public source on GitHub.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 14000,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "editorial_notes": "Added 2026-05-01 to fill the pyproject `[tool.*]` detection path so the audit no longer reports pyright as an unknown tool.\n",
    "inclusion_rationale": "Occupies detection × post_generation × fully_autonomous for logic_error and fabrication in the Python lane, alongside mypy. Distinct because Pyright targets editor-grade incremental performance and ships the language server most agentic IDE integrations rely on; missing it leaves the audit detector blind to a major slice of real Python configurations.\n"
  },
  {
    "id": "pyrit",
    "name": "PyRIT",
    "tagline": "The Python Risk Identification Tool for generative AI (PyRIT).",
    "description": "Microsoft PyRIT is an open-source framework for AI red-teaming. Automates probing of generative AI systems for risks (prompt injection, harmful content, jailbreaks) via a plug-in orchestrator with configurable attack and scoring strategies.\n",
    "first_released": "2023-12-12",
    "last_updated": "2026-04-23",
    "maturity_status": "experimental",
    "published_by": "microsoft",
    "repository_url": "https://github.com/microsoft/PyRIT",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "security_vulnerability",
      "fabrication",
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "red-team-orchestrator",
      "attack-strategies",
      "scoring-engines"
    ],
    "integration_surfaces": [
      "library",
      "cli"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 3740,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Microsoft-ecosystem red-team framework — complements garak/fuzzyai with an orchestration pattern that maps well onto azure-hosted workloads.\n"
  },
  {
    "id": "pytest",
    "name": "pytest",
    "tagline": "Python test framework — discipline gate for \"done means tests pass.\"",
    "description": "pytest is the de facto Python test framework: it discovers and runs unit, integration, and functional tests, reporting failures in human-readable form. It is the canonical discipline gate for the \"done means tests pass\" rule — surfacing incomplete execution (skipped/never-written tests), test_manipulation (xfail/skip abuse), and logic errors that only show up under exercise. Configuration lives in `pyproject.toml [tool.pytest.ini_options]`, which the audit detector now recognises.\n",
    "first_released": "2004-11-29",
    "last_updated": "2026-04-15",
    "maturity_status": "stable",
    "published_by": "python",
    "repository_url": "https://github.com/pytest-dev/pytest",
    "documentation_url": "https://docs.pytest.org/",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "incomplete_execution",
      "test_manipulation",
      "logic_error"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "test-discovery",
      "fixture-injection",
      "parametrized-testing",
      "assertion-introspection"
    ],
    "integration_surfaces": [
      "cli",
      "ci_app",
      "ide_plugin"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [
      "python"
    ],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Standard dev-tool security posture; pytest executes user-authored test code with full developer privileges, so test files should be reviewed with the same scrutiny as production code. Plugin ecosystem is large and worth pinning by version.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 12000,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "editorial_notes": "Added 2026-05-01 to fill the pyproject `[tool.*]` detection path so the audit no longer reports pytest as an unknown tool.\n",
    "inclusion_rationale": "Occupies detection × post_generation × fully_autonomous for the incomplete_execution and test_manipulation failure modes in the Python lane. Mature workhorse; missing it leaves the audit detector unable to recognise the single most common testing configuration in Python repos.\n"
  },
  {
    "id": "ragas",
    "name": "ragas",
    "tagline": "Supercharge your LLM application evaluations — RAG- and agent-centric metrics.",
    "description": "Ragas is an open-source evaluation framework for LLM applications with specialized metrics for RAG systems (faithfulness, context precision/recall, answer relevancy) and agent trajectories. Pytest-compatible, widely adopted in the RAG community, and frequently paired with tracing backends.\n",
    "first_released": "2023-05-08",
    "last_updated": "2026-02-24",
    "maturity_status": "stable",
    "published_by": "exploding-gradients",
    "repository_url": "https://github.com/explodinggradients/ragas",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "full_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error",
      "context_pollution"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "rag-metrics",
      "faithfulness",
      "context-recall",
      "context-precision"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 13605,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Canonical RAG-eval library. Complements deepeval (broad LLM eval) and autorag (pipeline search) by owning the faithfulness / context-precision metric family.\n"
  },
  {
    "id": "ragflow",
    "name": "ragflow",
    "tagline": "Open-source RAG engine based on deep document understanding.",
    "description": "RAGFlow is an open-source RAG engine with deep document understanding. Provides a production-grade RAG pipeline with OCR + layout-aware parsing, hybrid retrieval, and traceable generation. Reduces hallucinations by grounding answers in cited document chunks.\n",
    "first_released": "2023-12-12",
    "last_updated": "2026-04-23",
    "maturity_status": "stable",
    "published_by": "infiniflow",
    "repository_url": "https://github.com/infiniflow/ragflow",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "context_pollution"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "ocr-document-parsing",
      "hybrid-retrieval",
      "traceable-generation"
    ],
    "integration_surfaces": [
      "http_service",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 78820,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Production-RAG-pipeline cell with traceability — complements autorag (pipeline search) by focusing on deployable pipelines.\n"
  },
  {
    "id": "rebuff",
    "name": "Rebuff",
    "tagline": "LLM Prompt Injection Detector",
    "description": "Rebuff is a prompt injection detection system designed to identify and filter malicious injection attempts in LLM prompts before processing. It analyzes incoming prompts to detect adversarial patterns that could compromise application security or manipulate model behavior.",
    "first_released": "2023-04-24",
    "last_updated": "2024-08-07",
    "maturity_status": "at_risk",
    "repository_url": "https://github.com/protectai/rebuff",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "pre_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "security_vulnerability"
    ],
    "locus_of_control": [
      "prompt",
      "environment"
    ],
    "implements_techniques": [],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 1464,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies detection × pre_generation × fully_autonomous with a four-stage ensemble for prompt-injection defense: heuristics, a dedicated classifier LLM, a vector store of prior attacks, and canary-token leak detection on the outbound side. Distinguished from last-layer (pure low-latency classifier) by trading latency for richer defense-in-depth, and from llm-guard by being injection-specialist rather than a general PII/toxicity suite. Useful anchor for the ensemble-detector style of pre-generation defense.\n"
  },
  {
    "id": "renovate",
    "name": "Renovate",
    "tagline": "Highly configurable multi-platform dependency update bot.",
    "description": "Open-source dependency-update bot from Mend (formerly WhiteSource) that scans repositories across GitHub, GitLab, Bitbucket, Azure DevOps, and Gitea, then opens PRs/MRs to bump outdated packages. Supports 90+ package managers and offers fine-grained config for grouping, scheduling, automerge rules, and vulnerability-only alerts via a JSON/JSON5 renovate.json file. Self-hostable as a CLI or Docker image; a hosted Mend Renovate App is also available for GitHub.\n",
    "first_released": "2017-03-01",
    "last_updated": "2026-04-18",
    "maturity_status": "stable",
    "published_by": "mend",
    "repository_url": "https://github.com/renovatebot/renovate",
    "documentation_url": "https://docs.renovatebot.com",
    "license": "AGPL-3.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "obsolescence",
      "supply_chain_attack"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "dependency-version-diff",
      "cve-advisory-matching",
      "automated-pull-request",
      "config-driven-grouping",
      "scheduled-update-batches"
    ],
    "integration_surfaces": [
      "ci_app",
      "cli"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [
      "npm",
      "pypi",
      "maven",
      "go-modules",
      "cargo",
      "bundler",
      "composer",
      "nuget",
      "docker",
      "github-actions",
      "terraform"
    ],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "AGPL-3.0 core; the Mend-hosted app adds proprietary enrichment. Bot runs with repo write scope to open PRs; self-hosted deployments keep manifest data on-prem. Automerge configurations must be scoped (e.g., devDependencies patch only) so that compromised upstream releases are not merged without human review - the same supply-chain caveat applies as with any auto-updater.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 21353,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies detection x post_generation x graduated_hitl alongside Dependabot but with config-first ergonomics and cross-forge reach (GitLab, Bitbucket, Azure DevOps), filling the same MECE cell for teams that are not on GitHub or need update-grouping that Dependabot does not express. Its broader manager coverage (Terraform, Helm, GitHub Actions digests) extends obsolescence mitigation into infrastructure-as-code manifests that pure application-package tools miss.\n"
  },
  {
    "id": "ruff",
    "name": "Ruff",
    "tagline": "Extremely fast Python linter and formatter, written in Rust.",
    "description": "Ruff is a Rust-based linter and formatter for Python that consolidates the rule sets of flake8, isort, pydocstyle, pyupgrade, and many other legacy tools into a single binary that runs orders of magnitude faster. It runs from the CLI, in pre-commit, in CI, and as a language-server inside editors, surfacing logic errors, dead code, import ordering issues, and stylistic drift before review. Configuration lives in `pyproject.toml [tool.ruff]`, which is exactly the surface the audit detector now parses.\n",
    "first_released": "2022-08-30",
    "last_updated": "2026-04-15",
    "maturity_status": "stable",
    "published_by": "astral-sh",
    "repository_url": "https://github.com/astral-sh/ruff",
    "documentation_url": "https://docs.astral.sh/ruff/",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "logic_error",
      "scope_creep"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "lint-rule-engine",
      "code-formatting",
      "ast-walking",
      "import-sorting"
    ],
    "integration_surfaces": [
      "cli",
      "ci_app",
      "ide_plugin"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [
      "python"
    ],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Standard dev-tool security posture; runs offline against local source with no network calls. Rule definitions ship with the binary, so supply-chain risk is bounded to the pinned Ruff version itself.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 30000,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "editorial_notes": "Added 2026-05-01 to fill the pyproject `[tool.*]` detection path so the audit no longer reports ruff as an unknown tool.\n",
    "inclusion_rationale": "Occupies detection × post_generation × fully_autonomous for logic_error and scope_creep in the Python lane. Mature workhorse rather than novel research, but missing it leaves the audit detector blind to the most common `pyproject.toml [tool.ruff]` configuration in modern Python repos (e.g. openai-python).\n"
  },
  {
    "id": "safe-rlhf",
    "name": "safe-rlhf",
    "tagline": "Safe RLHF: constrained value alignment via safe reinforcement learning from human feedback.",
    "description": "PKU Alignment's Safe-RLHF is a reference implementation of Safe RLHF — reinforcement learning from human feedback with explicit safety constraints (constrained optimization). Used to train helpful-and-harmless LLMs with measurable safety bounds.\n",
    "first_released": "2023-05-15",
    "last_updated": "2025-11-24",
    "maturity_status": "experimental",
    "published_by": "pku-alignment",
    "repository_url": "https://github.com/PKU-Alignment/safe-rlhf",
    "license": "Apache-2.0",
    "control_paradigm": "prevention",
    "temporal_phase": "pre_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "security_vulnerability",
      "scope_creep",
      "context_pollution"
    ],
    "locus_of_control": [
      "model"
    ],
    "implements_techniques": [
      "constrained-rlhf",
      "helpful-harmless-training",
      "safety-preference-modeling"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 1598,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Training-time supervision: the safety-constrained RLHF branch of model alignment. Complements detection-time tools by preventing unsafe behaviour at the model layer rather than the runtime layer.\n"
  },
  {
    "id": "semgrep",
    "name": "Semgrep",
    "tagline": "Fast, rule-based static analysis with a pattern syntax that mirrors source code.",
    "description": "Open-source static-analysis engine that matches syntactic and lightweight-dataflow patterns written in a source-code-like DSL, spanning 30+ languages. Ships a large community rule registry (OWASP Top 10, secrets, framework anti-patterns) and supports custom rules that can flag reinvented-wheel patterns - e.g., hand-rolled retry loops, ad-hoc string-escaping, or reimplemented stdlib utilities - surfacing places where an agent wrote code that an existing library call would have covered. Runs in CLI, pre-commit, and CI; Semgrep AppSec Platform adds hosted findings triage.\n",
    "first_released": "2020-03-17",
    "last_updated": "2026-04-17",
    "maturity_status": "stable",
    "published_by": "semgrep",
    "repository_url": "https://github.com/semgrep/semgrep",
    "documentation_url": "https://semgrep.dev/docs/",
    "license": "LGPL-2.1",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "dependency_blindness",
      "security_vulnerability"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "pattern-based-static-analysis",
      "taint-analysis",
      "custom-rule-authoring",
      "ci-gating",
      "secret-scanning"
    ],
    "integration_surfaces": [
      "cli",
      "ci_app",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [
      "python",
      "javascript",
      "typescript",
      "go",
      "java",
      "ruby",
      "c",
      "cpp",
      "terraform"
    ],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Core engine is LGPL-2.1 and runs fully offline; the default rule registry is pulled from semgrep.dev and should be pinned by commit or vendored for regulated builds. The hosted AppSec Platform is SaaS and sends findings (not raw source by default) upstream; self-hosted deployments keep everything on-prem. Rules execute as pattern matching, not arbitrary code, so registry compromise is bounded to false-positive/negative impact rather than RCE.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 14901,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies detection x post_generation x fully_autonomous for dependency_blindness: custom rules catch patterns such as \"string built with manual concatenation where shlex.quote exists\" or \"home-rolled HTTP retry where tenacity is on the requirements\", pointing reviewers at library-duplicating code after the fact. Distinct from tree-sitter (structural substrate, prevention) and sourcegraph-cody (retrieval grounding) because Semgrep is rule-based diagnosis on the finished diff, making it the classic CI-gate node in the MECE grid.\n"
  },
  {
    "id": "sentrux",
    "name": "sentrux",
    "tagline": "Real-time architectural sensor that scores codebase modularity, acyclicity, and redundancy as the agent writes.",
    "description": "Pure-Rust CLI plus MCP server that watches a codebase's structural health in real time and emits a single 0–10000 quality signal computed from five root-cause metrics (modularity, acyclicity, depth, equality, redundancy). Ships an interactive treemap GUI, a CI-friendly rules engine (`.sentrux/rules.toml` declares max-cycles, layer order, forbidden boundaries, god-file thresholds) and a session-baseline gate (`sentrux gate`) that flags architectural regression between agent runs. MCP integration exposes nine tools (scan, health, session_start/end, rescan, check_rules, evolution, dsm, test_gaps), letting Claude Code, Cursor, Windsurf, and OpenCode read the sensor signal and self-correct without a human in the loop. 52 languages via tree-sitter plugins.\n",
    "first_released": "2026-03-11",
    "last_updated": "2026-03-19",
    "maturity_status": "experimental",
    "published_by": "sentrux",
    "repository_url": "https://github.com/sentrux/sentrux",
    "documentation_url": "https://github.com/sentrux/sentrux#readme",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "cascading_failure",
      "dependency_blindness"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "architectural-metrics",
      "dependency-cycle-detection",
      "cross-session-drift-tracking",
      "structural-rules-engine",
      "ci-gating"
    ],
    "integration_surfaces": [
      "cli",
      "mcp_server",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Pure Rust, single binary, runs fully local. Reads source files read-only; the MCP server exposes scan/session/rules tools but no filesystem write capability. Rules and constraints live as a human-authored `.sentrux/rules.toml` in the repo. No telemetry is documented in the README; verify before shipping into a regulated environment.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 1975,
    "stars_updated_at": "2026-05-03",
    "contributors": [],
    "inclusion_rationale": "Occupies detection × post_generation × fully_autonomous for cascading_failure (continuous architectural metrics let an agent detect that its own session regressed quality before the next edit) and dependency_blindness (the redundancy metric flags scattered re-implementations of the same concept across files). Distinct from langfuse / helicone (LLM-call observability) by observing the *codebase structure* rather than the model trace, and from semgrep / tree-sitter (rule-or-pattern static analysis) by emitting a continuous live score plus session-baseline diff rather than discrete pass/fail findings. The CI-gate mode shifts the same binary toward prevention and graduated_hitl; if that use-case grows we may split it into a second Tool node per the MECE policy.\n"
  },
  {
    "id": "skyvern",
    "name": "skyvern",
    "tagline": "Automate browser-based workflows with LLMs and computer vision.",
    "description": "Skyvern automates browser workflows using LLMs plus computer vision. Browser-use supervision primitives include selector-robust DOM grounding, screenshot reasoning, and structured action plans. Positioned as a business-process-automation layer on top of browser agents.\n",
    "first_released": "2024-02-28",
    "last_updated": "2026-04-23",
    "maturity_status": "stable",
    "published_by": "skyvern-ai",
    "repository_url": "https://github.com/Skyvern-AI/skyvern",
    "license": "AGPL-3.0",
    "control_paradigm": "prevention",
    "temporal_phase": "post_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "scope_creep",
      "security_vulnerability"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "cv-grounded-actions",
      "structured-plans",
      "selector-robustness"
    ],
    "integration_surfaces": [
      "library",
      "http_service"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 21342,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Browser-agent w/ CV-grounding cell. Differentiated from browser-use (DOM-only) by treating screenshots as primary.\n"
  },
  {
    "id": "smolagents",
    "name": "Smolagents",
    "tagline": "Minimalist code-agent library that expresses actions as executed Python.",
    "description": "Hugging Face's minimalist agent framework (≈thousand lines of code) built around CodeAgent, which has the LLM emit Python snippets that are executed against a controlled tool namespace rather than producing JSON tool-call objects. Supervision primitives: a sandboxed Python executor (local or E2B/Docker), an explicit authorized-imports list, planning-step hooks, and trajectory inspection. Action-as-code gives richer composition but shifts the surface from JSON-schema validation to Python-sandbox integrity.\n",
    "first_released": "2024-12-31",
    "last_updated": "2026-03-25",
    "maturity_status": "stable",
    "published_by": "huggingface",
    "repository_url": "https://github.com/huggingface/smolagents",
    "documentation_url": "https://huggingface.co/docs/smolagents",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "logic_error",
      "fabrication"
    ],
    "locus_of_control": [
      "environment",
      "prompt"
    ],
    "implements_techniques": [
      "code-as-action",
      "authorized-imports-list",
      "python-sandbox",
      "planning-step-hook"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Open-source (Apache-2.0). CodeAgent executes model-emitted Python — the default local executor only whitelists imports and is not a hard sandbox; for untrusted prompts use the E2B or Docker executor. Review the authorized-imports list per deployment; a broad list effectively grants the model arbitrary local-Python capability.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 26835,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies detection × post_generation × fully_autonomous for the \"code-as-action\" agent archetype. Distinct enough from JSON-tool-call frameworks that its supervision primitives (authorized-imports, Python sandbox) are worth modeling separately.\n"
  },
  {
    "id": "snyk",
    "name": "Snyk",
    "tagline": "Vulnerability scanning for code, dependencies, containers, and IaC.",
    "description": "Commercial developer-security platform with four product lines: Snyk Code (SAST via DeepCode-derived analysis), Snyk Open Source (dependency CVE scanning and SBOM), Snyk Container, and Snyk IaC. Integrates via CLI, IDE plugins, Git-provider apps, and CI gates. Supervision locus is the committed artifact: scans run against the repo or build and report known vulnerabilities with fix advice. Includes a proprietary vulnerability DB that complements public advisory feeds.\n",
    "first_released": "2015-04-01",
    "last_updated": "2026-04-12",
    "maturity_status": "stable",
    "published_by": "snyk",
    "repository_url": "https://github.com/snyk/snyk",
    "documentation_url": "https://docs.snyk.io",
    "license": "proprietary",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "security_vulnerability",
      "supply_chain_attack"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "static-analysis",
      "dependency-cve-scan",
      "sbom-generation",
      "container-scan",
      "iac-policy-scan"
    ],
    "integration_surfaces": [
      "cli",
      "ci_app"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [
      "npm",
      "pypi",
      "maven",
      "go-modules",
      "container-images"
    ],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Proprietary hosted platform with a free tier; scan results and dependency graphs are sent to Snyk's backend. Self-hosted broker is available for higher-trust environments. The Snyk CLI is distributed via npm and native installers; pin to a vetted release. Known-CVE scanning is inherently backward-looking — it does not catch novel malicious packages the way manifest-capability analysis does.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 5506,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies detection × post_generation × fully_autonomous for the security-vulnerability failure mode. Pairs naturally with Socket in the taxonomy: Socket screens adoption (pre_generation manifest gating), Snyk scans what already landed (post_generation artifact analysis). Both are supply-chain-relevant but at different moments.\n"
  },
  {
    "id": "socket",
    "name": "Socket",
    "tagline": "Dependency supply-chain scanner that inspects packages before install.",
    "description": "Commercial dependency-security platform that analyzes packages in real time for supply-chain risks (typosquats, install-time scripts, network/file/shell capability changes, obfuscation, sudden maintainer churn, known malware). Integrates as a GitHub App, CLI, and package-manager wrapper (socket-npm/pip) that evaluates dependencies before install lands. Supervision locus is the manifest (package.json, requirements.txt, go.mod): risks are flagged pre-install rather than relying on known-CVE databases alone.\n",
    "first_released": "2022-06-01",
    "last_updated": "2026-04-08",
    "maturity_status": "stable",
    "published_by": "socket-dev",
    "repository_url": "https://github.com/SocketDev",
    "documentation_url": "https://docs.socket.dev",
    "license": "proprietary",
    "control_paradigm": "detection",
    "temporal_phase": "pre_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "supply_chain_attack"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "dependency-manifest-scan",
      "capability-diff",
      "typosquat-detection",
      "install-script-analysis",
      "pull-request-gating"
    ],
    "integration_surfaces": [
      "ci_app",
      "cli",
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [
      "npm",
      "pypi",
      "go-modules"
    ],
    "cited_in": [
      "spracklen-2024-we-have-a-package"
    ],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "Proprietary hosted platform. Dependency metadata — and, for some analyses, source — is sent to Socket's backend for scanning; review data-handling terms before enabling on private code. The GitHub App requires repo read permissions. CLI and wrappers are distributed via npm/pip; pin versions to audited releases.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "contributors": [],
    "inclusion_rationale": "Occupies detection × pre_generation × fully_autonomous for the supply-chain-attack failure mode. Scanning manifests before install is the specific supervision primitive that separates Socket from post-install vulnerability scanners — it catches slopsquatted and typosquatted packages at the moment of adoption, which is when coding agents are most likely to introduce them.\n"
  },
  {
    "id": "superagent",
    "name": "Superagent",
    "tagline": "Security layer protecting AI apps from prompt injection, data leaks, and harmful outputs.",
    "description": "A security wrapper that protects AI applications against prompt injections, data leaks, and harmful outputs. Developers embed this safety layer directly into their applications to enforce runtime security policies and demonstrate compliance.",
    "first_released": "2023-05-10",
    "last_updated": "2026-04-11",
    "maturity_status": "experimental",
    "repository_url": "https://github.com/superagent-ai/superagent",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "in_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "security_vulnerability"
    ],
    "locus_of_control": [
      "prompt",
      "environment"
    ],
    "implements_techniques": [],
    "integration_surfaces": [
      "library",
      "http_service"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 6553,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies prevention × in_generation × fully_autonomous as an embedded security wrapper: developers link the library directly into the application so policy enforcement runs inline with the LLM call rather than behind a network proxy (contrast pipelock) or as a post-hoc validator (contrast guardrails-ai). Emphasizes compliance-ready runtime enforcement, distinguishing it from the detector-only posture of rebuff and last-layer, and making it a useful exemplar of in-process guardrail deployment.\n"
  },
  {
    "id": "tree-sitter",
    "name": "tree-sitter",
    "tagline": "Incremental parser toolkit that powers structural code analysis and symbol discovery.",
    "description": "Parser generator and incremental parsing library that produces concrete syntax trees for 100+ languages fast enough to reparse on every keystroke. Used as the structural-analysis substrate inside editors (Neovim, Helix, Zed), language servers, and code-intelligence pipelines - including duplicate-function detection, symbol extraction for RAG, and dead-code/unused-import linters. Ships as a C core plus language-specific grammar crates and bindings for Rust, Python, Node, Go, and Swift.\n",
    "first_released": "2018-10-01",
    "last_updated": "2026-04-10",
    "maturity_status": "stable",
    "published_by": "tree-sitter",
    "repository_url": "https://github.com/tree-sitter/tree-sitter",
    "documentation_url": "https://tree-sitter.github.io/tree-sitter/",
    "license": "MIT",
    "control_paradigm": "prevention",
    "temporal_phase": "pre_generation",
    "autonomy_level": "fully_autonomous",
    "addresses_failure_modes": [
      "dependency_blindness"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "incremental-parsing",
      "concrete-syntax-tree",
      "query-based-symbol-extraction",
      "structural-code-search"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [
      "javascript",
      "typescript",
      "python",
      "go",
      "rust",
      "java",
      "c",
      "cpp",
      "ruby"
    ],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "security_notes": "MIT-licensed C library; grammars are community-maintained and compiled at build time. Running grammars on untrusted source is safe (pure parsing, no execution), but grammar-generation from .grammar.js uses node-gyp and should be pinned to audited grammar revisions to avoid supply-chain drift. No network access.\n",
    "signed_manifest": false,
    "openssf_scorecard_score": null,
    "stars": 24948,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies prevention x pre_generation x fully_autonomous for the dependency_blindness failure mode by providing the structural index that surfaces existing symbols, functions, and module exports before an agent reinvents them. Distinctive because it is the substrate many higher-level tools (LSPs, Sourcegraph SCIP emitters, grep-plus-AST linters) build on, which makes it the lowest-level entry point in the MECE grid for structural prevention - complementary to Semgrep (detection via custom rules) and sourcegraph-cody (retrieval grounding).\n"
  },
  {
    "id": "trulens",
    "name": "TruLens",
    "tagline": "Evaluation and Tracking for LLM Experiments and AI Agents",
    "description": "TruLens provides open-source tools for evaluating and tracking LLM experiments with programmable feedback functions. It enables developers to instrument, evaluate, and iterate on LLM applications using feedback functions for quality assessment, including coverage of context relevance, groundedness, and answer relevance.",
    "first_released": "2020-11-02",
    "last_updated": "2026-04-22",
    "maturity_status": "experimental",
    "repository_url": "https://github.com/truera/trulens",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "context_pollution"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 3266,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies detection × post_generation × graduated_hitl with the RAG Triad — context relevance, groundedness, and answer relevance — as programmable feedback functions over instrumented traces. Distinguished from general LLM-observability platforms (langfuse, langsmith, arize-phoenix) by centering RAG-specific groundedness metrics and from deepeval by exposing those metrics as live feedback functions rather than Pytest assertions. Useful as the canonical RAG-evaluation exemplar in the post-generation detection cell.\n"
  },
  {
    "id": "uqlm",
    "name": "uqlm",
    "tagline": "Uncertainty Quantification for Language Models.",
    "description": "UQLM (by CVS Health) is a Python package for uncertainty quantification of LLM outputs. Implements black-box, white-box, and semantic-similarity-based uncertainty estimators so downstream code can abstain, escalate to human review, or request clarification when confidence is low.\n",
    "first_released": "2025-04-17",
    "last_updated": "2026-04-16",
    "maturity_status": "experimental",
    "published_by": "cvs-health",
    "repository_url": "https://github.com/cvs-health/uqlm",
    "license": "Apache-2.0",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "fabrication",
      "logic_error"
    ],
    "locus_of_control": [
      "model",
      "environment"
    ],
    "implements_techniques": [
      "black-box-uq",
      "white-box-uq",
      "semantic-uq",
      "abstain-routing"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 1146,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Uncertainty-first branch of detection — enables explicit abstain/escalate routing, valuable for high-stakes medical/finance LLM deployments.\n"
  },
  {
    "id": "voltagent",
    "name": "voltagent",
    "tagline": "TypeScript-native AI agent engineering platform with built-in observability.",
    "description": "VoltAgent is a TypeScript-native AI agent framework with tightly integrated observability (VoltOps console). Provides workflow primitives and a UI for inspecting agent runs. Positions itself as the TS alternative to LangChain/agents with observability built in.\n",
    "first_released": "2025-04-16",
    "last_updated": "2026-04-23",
    "maturity_status": "stable",
    "published_by": "voltagent",
    "repository_url": "https://github.com/VoltAgent/voltagent",
    "license": "MIT",
    "control_paradigm": "detection",
    "temporal_phase": "post_generation",
    "autonomy_level": "graduated_hitl",
    "addresses_failure_modes": [
      "scope_creep",
      "context_pollution",
      "incomplete_execution"
    ],
    "locus_of_control": [
      "environment"
    ],
    "implements_techniques": [
      "ts-agent-framework",
      "builtin-observability",
      "workflow-primitives"
    ],
    "integration_surfaces": [
      "library"
    ],
    "composes_with": [],
    "feeds_into": [],
    "supersedes": [],
    "runtime_requires": [],
    "supervises_targets": [],
    "cited_in": [],
    "documented_in": [],
    "evaluated_on": [],
    "openssf_scorecard_score": null,
    "stars": 8416,
    "stars_updated_at": "2026-04-23",
    "contributors": [],
    "inclusion_rationale": "Occupies the TypeScript-agent-framework-with-builtin-observability point. Complements mastra by leaning more into run-time inspection tooling.\n"
  }
]