{ "built_at": "2026-09-15T00:00:00Z", "version": 2, "rules": "RULES.md v0.1 (2026-09-15)", "default_policy": "standard", "policies": { "leads": { "label": "Leads included", "short": "everything", "desc": "Every signal that is not quarantined counts, including imported and unverifiable leads, which can support only a 2. For research, not for citation." }, "standard": { "label": "Standard", "short": "confirmed sources", "desc": "A signal counts only if at least one cited source was re-fetched and confirmed. The default: no number moves on a source you cannot open and check." }, "against_interest": { "label": "Against interest", "short": "no self-serving self-report", "desc": "Standard, and an organization's own statements count only when they are against its interest or backed by a non-self source." }, "spans": { "label": "Verified spans", "short": "quoted spans only", "desc": "Standard, and the signal carries a quoted span you can find on the page." }, "primary": { "label": "Primary only", "short": "filings and indexes", "desc": "Only confirmed filings, funder indexes, or third-party ledgers with a quoted span: what can be verified from outside the field." } }, "policy_order": [ "leads", "standard", "against_interest", "spans", "primary" ], "bands": { "clear": { "label": "Clear", "desc": "No evidenced dimension below 2." }, "conditional": { "label": "Conditional floor", "desc": "At least one evidenced dimension at 1: usable with conditions the card names." }, "disqualifying": { "label": "Disqualifying floor", "desc": "At least one evidenced dimension at 0: not a candidate for an independence-critical role until the floor moves." }, "unevidenced": { "label": "Unevidenced", "desc": "No dimension is evidenced under this policy." } }, "roles": { "referee": "Independent referee", "government": "Government institute", "vendor": "Vendor / red-team co.", "benchmark": "Benchmark or consortium", "lab-team": "First-party lab team", "expected-entrant": "Expected entrant" }, "groups": { "referee": { "label": "Independent referees", "desc": "Nonprofits, public benefit corporations, and academic groups whose work is evaluation or research and who sell nothing to the labs they grade. Lab fees for the evaluation itself keep an organization here; they lower its funding value instead." }, "government": { "label": "Government institutes", "desc": "Public bodies. Publication and access are scored on outcome and tagged statutory where the constraint is the law, not a lab." }, "commercial": { "label": "Commercial and first-party", "desc": "Vendors, first-party lab teams, and consortia funded by the labs they benchmark. Competent testers can sit here; the rubric measures structure, not skill." } }, "group_order": [ "referee", "government", "commercial" ], "dimensions": [ { "key": "F", "label": "Funding", "desc": "Where the money comes from, and whether any of it comes from the developers being evaluated or their investors.", "anchors": [ "Owned or controlled by a frontier developer or a lab investor: a stake of 20% or more, or a business unit.", "Material revenue or investment from evaluated labs or their investors.", "Labs pay per engagement; otherwise diversified.", "Mostly philanthropic or public money; some lab-linked pooled funds.", "No lab money; diversified philanthropic or public funding, disclosed." ] }, { "key": "G", "label": "Governance", "desc": "Legal form, board, and whether a conflict-of-interest policy is published.", "anchors": [ "Unit or subsidiary of a lab or a lab's investor.", "VC-backed for-profit with no published COI policy.", "For-profit or PBC with a published COI policy.", "Nonprofit or public body with a COI policy.", "Nonprofit or public body, published COI policy, independent board, external review." ] }, { "key": "P", "label": "Personnel", "desc": "Board seats, equity, advisory roles, and the revolving door between evaluator and lab.", "anchors": [ "Leaders hold governance roles at an evaluated lab, no recusal.", "Leaders hold equity or advisory roles at labs; informal recusal.", "Frequent two-way hiring; recusal on request.", "Recusal policy and disclosure of lab ties.", "Cooling-off periods, disclosed ties, no equity in labs." ] }, { "key": "A", "label": "Access depth (lab-granted)", "desc": "The deepest access labs have actually granted this evaluator in practice: public API, pre-release API, safeguards off, weights and logs, or embedded during training. This is granted access, not an institutional right or a capability measure: a low score can mean labs did not grant access, not that the evaluator lacks competence. It records who labs chose to let in.", "anchors": [ "Public API only.", "Pre-release API with safeguards on.", "Pre-release with safeguards off or extended time.", "Helpful-only or weights-level access, chain of thought, logs, on-site.", "Embedded, training-time, or incident access." ] }, { "key": "S", "label": "Scope control", "desc": "Who decides what gets tested, for how long, and whether the evaluator can refuse to sign off.", "anchors": [ "Lab defines tasks and can decline findings.", "Lab defines scope; evaluator picks methods.", "Scope negotiated per engagement.", "Evaluator sets scope and can add questions.", "Evaluator sets scope, can investigate incidents, can refuse sign-off." ] }, { "key": "R", "label": "Publication rights", "desc": "Whether findings reach the public unedited, and whether adverse findings have been published.", "anchors": [ "No publication, or lab approval required.", "Lab-edited summaries only.", "Publishes; lab reviews with broad redaction.", "Publishes; redaction limited to security; redactions disclosed.", "Full editorial control, record of adverse findings, redaction statements." ] }, { "key": "M", "label": "Method transparency", "desc": "Whether evaluation code, tasks, and conditions are open enough for others to reproduce.", "anchors": [ "Closed.", "Summaries only.", "Methods described in prose.", "Tasks or code partly open.", "Open code, tasks, reproducible runs, factsheets." ] }, { "key": "X", "label": "Role incompatibility", "desc": "Whether the organization both grades labs and sells to them — the audit-plus-consulting problem. Selling products or services to an evaluated lab is a role conflict, not proof that any given evaluation was wrong; a vendor can be a competent tester and still be structurally compromised as a referee.", "anchors": [ "Sells defense or monitoring products to evaluated labs.", "Sells products or services other than the evaluation itself to labs or to their customers.", "Consults for labs.", "Tools are open or free to the ecosystem.", "No commercial products." ] } ], "presets": { "lab": { "label": "Lab procurement", "weights": { "F": 18, "G": 8, "P": 10, "A": 20, "S": 16, "R": 14, "M": 9, "X": 5 }, "derivation": "Persona: a lab procurement lead choosing an outside evaluator whose report will be cited in a system card. Weights follow what a reader of that card will test the report against: access depth (20) and scope control (16) because a shallow or lab-scoped engagement is the first thing a critic checks; funding (18) and publication rights (14) because AEF-1's operating conditions and Illinois SB 315's financial-interest test both turn on them; personnel (10), method transparency (9), governance (8), and role incompatibility (5) round it out. Fixed in bench/seed_v0.py on 15 September 2026 before the population pass of the same day; not registered outside this repository, so it is the confirmatory preset, not a pre-registered one." }, "regulator": { "label": "Regulator or auditor", "weights": { "F": 15, "G": 10, "P": 10, "A": 20, "S": 20, "R": 15, "M": 10, "X": 0 }, "derivation": "Persona: a regulator or accredited auditor selecting an evaluator under a mandate. Scope control and access rise to 20 each because a regulator can compel both and will want to see them exercised; publication rights (15) and funding (15) follow the financial-audit independence rules SB 315 imports; governance (10), personnel (10), and methods (10) are the usual accreditation checks. Role incompatibility is 0 here because an audit mandate can forbid product sales outright rather than score them." }, "public": { "label": "Public trust", "weights": { "F": 25, "G": 15, "P": 15, "A": 5, "S": 10, "R": 20, "M": 5, "X": 5 }, "derivation": "Persona: a reader deciding whether to trust a published finding. Funding (25), publication rights (20), governance (15), and personnel (15) dominate because the public cannot inspect an engagement and can only ask who paid, who could edit, and who sits where. Scope (10), access (5), methods (5), and role incompatibility (5) matter less to trust in a finding that has already been published." }, "equal": { "label": "Equal weights", "weights": { "F": 12.5, "G": 12.5, "P": 12.5, "A": 12.5, "S": 12.5, "R": 12.5, "M": 12.5, "X": 12.5 }, "derivation": "Every dimension at 12.5. A sensitivity check that asks how much the ranking depends on any persona's priorities; it is not a claim that the dimensions matter equally." } }, "types": { "nonprofit": "Nonprofit", "pbc": "Public benefit corp", "vc": "VC-backed", "gov": "Government", "academic": "Academic", "bigtech": "Big Tech unit", "consortium": "Consortium", "private": "Private company, capital structure undisclosed", "hypothetical": "Hypothetical composite, not a real organization" }, "sources": { "80k-funding": { "audit_status": "confirmed", "id": "80k-funding", "note": "CAIS advises xAI, not receiving OP money; SFF grants; METR not receiving OP recently; SecureBio SFF grant. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches", "published": "2025-10-13", "publisher": "80,000 Hours", "retrieved": "2026-09-14", "source_type": "press", "title": "It looks like there are some good funding opportunities in AI safety right now", "url": "https://80000hours.org/2025/01/it-looks-like-there-are-some-good-funding-opportunities-in-ai-safety-right-now/", "press_kind": "primary" }, "aef-launch": { "audit_status": "confirmed", "id": "aef-launch", "note": "Founding members: Transluce, METR, RAND, HAL, SecureBio, CIP, AVERI/Brundage. AEF-1 minimum operating conditions. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches", "published": "2025-12-04", "publisher": "AI Evaluator Forum", "retrieved": "2026-09-14", "source_type": "self", "title": "AI Evaluator Forum launch and AEF-1", "url": "https://aievaluatorforum.org/", "self_of": "aef" }, "aef-transparency": { "audit_status": "confirmed", "id": "aef-transparency", "note": "Editorial control, access, and transparency disclosures expected of third-party evaluators. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026", "publisher": "AI Evaluator Forum", "retrieved": "2026-09-14", "source_type": "self", "title": "Evaluation Transparency Letter", "url": "https://t.co/Nm0RMxOWzw", "self_of": "aef" }, "aisi-alignment-60": { "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "id": "aisi-alignment-60", "note": "£12m additional funding bringing the Alignment Project to £27m, including £5.6m from OpenAI. [verify 2026-09-15] fetched; page reachable, citation matches; 1 quoted span(s) found verbatim", "published": "2026-02", "publisher": "UK AI Security Institute", "retrieved": "2026-09-15", "source_type": "self", "title": "Funding 60 projects to advance AI alignment research", "url": "https://www.aisi.gov.uk/blog/funding-60-projects-to-advance-ai-alignment-research", "self_of": "ukaisi" }, "aisi-grants": { "audit_status": "confirmed", "id": "aisi-grants", "imported_from": "kevinnbass/metr-money-figure:evaluators.csv EV11", "note": "GBP 27m coalition incl. GBP 5.6m from OpenAI plus Anthropic, Microsoft, AWS and others; awards up to GBP 1m per project. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2025", "publisher": "UK AI Security Institute", "retrieved": "2026-09-14", "source_type": "self", "title": "Alignment Project grants", "url": "https://www.aisi.gov.uk/grants", "self_of": "ukaisi" }, "aiwiki-ukaisi": { "audit_status": "confirmed", "id": "aiwiki-ukaisi", "note": "Microsoft parallel agreements May 2026; Cohere MOU; publication terms. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026-05-07", "publisher": "AI Wiki", "retrieved": "2026-09-14", "source_type": "press", "title": "UK AI Security Institute", "url": "https://aiwiki.ai/wiki/uk_aisi", "press_kind": "aggregator" }, "alignment-project-about": { "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "id": "alignment-project-about", "note": "Backers: UK AISI, Canadian AISI, CIFAR, Australian AISI, OpenAI, Microsoft, AWS, Anthropic, Schmidt Sciences, AI Safety Tactical Opportunities Fund, Halcyon Futures, Safe AI Fund, Sympatico Ventures, UKRI, ARIA; administered with Renaissance Philanthropy. [verify 2026-09-15] reachable BUT span(s) not found: ukaisi.12 — needs curator review [verify 2026-09-15 grok-pass] Fund >GBP 27M; backers incl. OpenAI, Microsoft, AWS, Anthropic, Schmidt, AISI/CAISI/CIFAR; no per-backer split.", "published": "2026", "publisher": "UK AI Security Institute", "retrieved": "2026-09-15", "source_type": "self", "title": "About the Alignment Project", "url": "https://alignmentproject.aisi.gov.uk/about", "self_of": "ukaisi" }, "alphasignal-metr-71m": { "audit_status": "confirmed", "id": "alphasignal-metr-71m", "note": "Reports ~$71M in commitments from foundations and individuals; hard rule on lab money. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026-08", "publisher": "AlphaSignal", "retrieved": "2026-09-14", "source_type": "press", "title": "METR raises $71M to independently stress-test AI", "url": "https://alphasignal.ai/news/metr-raises-71m-to-independently-stress-test-the-world-s-most-powerful-ai", "press_kind": "aggregator" }, "andon-fortune": { "id": "andon-fortune", "title": "Anthropic's office launched an AI-run vending machine. It evolved into AI-run stores and cafes within a year", "publisher": "Fortune", "url": "https://fortune.com/2026/06/02/anthropic-office-vending-machine-ai-agents-vendo-andon-lukas-petersson/", "published": "2026-06-02", "retrieved": "2026-09-15", "source_type": "press", "audit_status": "confirmed", "note": "Named outlet; co-founder Lukas Petersson quoted on the Anthropic deployment (Vendo). Does not mention xAI. Fetched 2026-09-15; span present in the article metadata.", "press_kind": "primary" }, "andon-pion": { "id": "andon-pion", "title": "Pion | Andon Labs", "publisher": "Andon Labs", "url": "https://andonlabs.com/pion", "published": "2026-09", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Pion research preview: \"Agents that run any company fully autonomously\"; waitlist. Fetched 2026-09-15; span present.", "self_of": "andon" }, "andon-pitchbook": { "audit_status": "confirmed", "id": "andon-pitchbook", "note": "About $500K raised; seven investors listed; conflicts with another database that lists no funding. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026", "publisher": "PitchBook", "retrieved": "2026-09-14", "source_type": "press", "title": "Andon Labs profile", "url": "https://pitchbook.com/profiles/company/541549-09", "press_kind": "aggregator" }, "andon-project-vend": { "id": "andon-project-vend", "title": "Project Vend: Can Claude run a small shop? (And why does that matter?)", "publisher": "Anthropic", "url": "https://www.anthropic.com/research/project-vend-1", "published": "2025-06-27", "retrieved": "2026-09-15", "source_type": "press", "audit_status": "confirmed", "note": "Anthropic's research post: \"Anthropic partnered with Andon Labs, an AI safety evaluation company, to have Claude Sonnet 3.7 operate a small, automated store in the Anthropic office\". The evaluated lab's document; typed press primary for want of a lab-document type. Fetched 2026-09-15; span present (normalizer inserts a space before the comma after \"Andon Labs\").", "press_kind": "primary" }, "andon-publications": { "id": "andon-publications", "title": "Publications", "publisher": "Andon Labs", "url": "https://andonlabs.com/publications", "published": "2026-09-14", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "List of blog posts and papers including \"Opus 5 on Vending-Bench: Once Again the Best Capitalist, Once Again Misaligned\" (2026-07-28) and \"Fable 5 on Vending-Bench: Misbehaving, with Plausible Deniability\" (2026-06-09). Fetched 2026-09-15; span present.", "self_of": "andon" }, "andon-site": { "audit_status": "confirmed", "id": "andon-site", "note": "Vending-Bench, Pion platform, autonomous deployments; Opus 5 results. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches", "published": "2026-07", "publisher": "Andon Labs", "retrieved": "2026-09-14", "source_type": "self", "title": "Andon Labs", "url": "https://andonlabs.com/", "self_of": "andon" }, "andon-vending-paper": { "id": "andon-vending-paper", "title": "Vending-Bench: A Benchmark for Long-Term Coherence of Autonomous Agents (arXiv 2502.15840)", "publisher": "Andon Labs (arXiv)", "url": "https://arxiv.org/abs/2502.15840", "published": "2025-02-20", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Andon's own paper describing the simulated vending environment. Abstract page; no code link. Fetched 2026-09-15; span present.", "self_of": "andon" }, "andon-yc": { "id": "andon-yc", "title": "Andon Labs: Autonomous organizations without humans in the loop", "publisher": "Y Combinator (company directory)", "url": "https://www.ycombinator.com/companies/andon-labs", "published": "2026", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "YC directory entry: Winter 2024 batch, Active, San Francisco. Company-authored listing on the investor's site; typed self. Fetched 2026-09-15; span present.", "self_of": "yc" }, "anthropic-alignment-faking": { "id": "anthropic-alignment-faking", "title": "Alignment faking in large language models", "publisher": "Anthropic", "url": "https://www.anthropic.com/research/alignment-faking", "published": "2024-12-18", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Lab-published research page (self-published by Anthropic, a third party to Redwood): \"A new paper from Anthropic's Alignment Science team, in collaboration with Redwood Research\"; acknowledgements name the collaboration; four independent reviewers.", "self_of": "anthropic" }, "anthropic-amodei-x-pace-frontier": { "id": "anthropic-amodei-x-pace-frontier", "title": "Dario Amodei on X: 'We Must Pace the Frontier ... We'll provide third-party evaluators with permanent, employee-level access'", "publisher": "Dario Amodei (Anthropic CEO), X", "url": "https://x.com/DarioAmodei/status/2098773920774074715", "published": "2026-09-12", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Announcement post (4:30 PM, Sep 12, 2026 per the page). fetch_text reads x.com status pages through their server-rendered title/OG text, so CI can check these spans (SHA-256 a9e72bbe0e27...). This is the 'employee-level' wording the paper uses; METR is not named in the post itself, only in the essay it links.", "self_of": "anthropic" }, "anthropic-pace-frontier": { "id": "anthropic-pace-frontier", "title": "We Must Pace the Frontier", "publisher": "Dario Amodei (Anthropic CEO), darioamodei.com", "url": "https://darioamodei.com/post/we-must-pace-the-frontier", "published": "2026-09-12", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "The essay behind the 'Pace the Frontier' plan. It is hosted on the CEO's personal domain, not anthropic.com; no anthropic.com post on the commitment was found by site search or URL probing as of 2026-09-15 (anthropic.com/news lists only an Aug 2026 alignment/security post that says Anthropic is 'planning to work with METR for an independent review'). The page carries no date metadata; the date is taken from the announcement post x.com/DarioAmodei/status/2098773920774074715 (2026-09-12), which links to it. Fetched 2026-09-15 with bench.review.fetch_text, SHA-256 62b8af679fe1...; the essay says 'employee-like', the X post says 'employee-level'. The space before ')' in the METR span comes from the normalizer stripping the link tag.", "self_of": "anthropic" }, "anthropic-series-a": { "id": "anthropic-series-a", "title": "Anthropic raises $124 million Series A", "publisher": "Anthropic", "url": "https://www.anthropic.com/news/anthropic-raises-124-million-to-build-more-reliable-general-ai-systems", "published": "2021-05-28", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Series A led by Jaan Tallinn with participation from Dustin Moskovitz. Self-published by the lab, not by any evaluator; used as the primary record that the principals of SFF (Tallinn) and Coefficient Giving (Moskovitz) are Anthropic investors. Shared record: identical in the palisade, cais, rand and hal files; ledger rows R01 and R02 already cite it.", "self_of": "anthropic" }, "apollo-about": { "audit_status": "confirmed", "id": "apollo-about", "note": "Ran evaluations for all major labs; partnered with OpenAI on anti-scheming; Watcher product. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026", "publisher": "Apollo Research", "retrieved": "2026-09-14", "source_type": "self", "title": "About", "url": "https://www.apolloresearch.ai/about", "self_of": "apollo" }, "apollo-first-year": { "audit_status": "confirmed", "id": "apollo-first-year", "note": "Contracted by UK AISI for deception evals; red-teamed OpenAI fine-tuning API. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2024", "publisher": "Apollo Research", "retrieved": "2026-09-14", "source_type": "self", "title": "The First Year of Apollo Research", "url": "https://www.apolloresearch.ai/blog/the-first-year-of-apollo-research", "self_of": "apollo" }, "apollo-manifund": { "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "id": "apollo-manifund", "note": "Self-report: about $1.5M from Open Philanthropy and about $500K from SFF at the time of the listing; date of the figures not stated. [verify 2026-09-15] fetched; page reachable, citation matches; 1 quoted span(s) found verbatim", "published": "2024", "publisher": "Manifund", "retrieved": "2026-09-15", "source_type": "self", "title": "Apollo Research on Manifund", "url": "https://manifund.org/apolloresearch", "self_of": "apollo" }, "apollo-norms": { "audit_status": "confirmed", "id": "apollo-norms", "note": "Four COI rules: no contingent compensation; no side grants/investments from evaluated orgs; recusal; no misrepresentation. States labs pay fair market value. [verify 2026-09-15] fetched; page reachable, citation matches; 1 quoted span(s) found verbatim", "published": "2025-11-26", "publisher": "Apollo Research", "retrieved": "2026-09-14", "source_type": "self", "title": "Our Norms on Security, Science Communication and Conflicts of Interest", "url": "https://www.apolloresearch.ai/blog/our-norms-coi-security-science-communication/", "self_of": "apollo" }, "apollo-pbc-mission": { "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "id": "apollo-pbc-mission", "note": "Mission seats on the board held by directors independent of the PBC and its funders. [verify 2026-09-15] fetched; page reachable, citation matches; 1 quoted span(s) found verbatim", "published": "2026-01-20", "publisher": "Apollo Research", "retrieved": "2026-09-15", "source_type": "self", "title": "Apollo Research is becoming a PBC (governance)", "url": "https://www.apolloresearch.ai/blog/apollo-research-is-becoming-a-pbc", "self_of": "apollo" }, "apollo-pbc-round": { "audit_status": "confirmed", "id": "apollo-pbc-round", "note": "Seed led by 50Y with Juniper, Macroscopic, Common Metal, SAIF, Ocean, Progress Fund, individuals; mission board seats. [verify 2026-09-15] fetched; page reachable, citation matches; 1 quoted span(s) found verbatim", "published": "2026-01-20", "publisher": "Apollo Research", "retrieved": "2026-09-14", "source_type": "self", "title": "Apollo Research is becoming a PBC (investment section)", "url": "https://www.apolloresearch.ai/blog/apollo-research-is-becoming-a-pbc", "self_of": "apollo" }, "apollo-pbc": { "audit_status": "confirmed", "id": "apollo-pbc", "note": "Spin-out from fiscal sponsor into public benefit corporation. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches", "published": "2026-01-20", "publisher": "Apollo Research", "retrieved": "2026-09-14", "source_type": "self", "title": "Apollo Research is becoming a PBC", "url": "https://www.apolloresearch.ai/blog/apollo-research-is-becoming-a-pbc", "self_of": "apollo" }, "arxiv-access-taxonomy": { "audit_status": "confirmed", "id": "arxiv-access-taxonomy", "note": "Taxonomy of access levels for external evaluators. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026-01-21", "publisher": "arXiv", "retrieved": "2026-09-14", "source_type": "press", "title": "Expanding External Access to Frontier AI Models for Dangerous Capability Evaluations (arXiv 2601.11916)", "url": "https://arxiv.org/abs/2601.11916", "press_kind": "primary" }, "arxiv-frontier-auditing": { "audit_status": "confirmed", "id": "arxiv-frontier-auditing", "note": "AI Assurance Levels; cross-industry precedents (FAA, UL, melamine, HackerOne). Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches", "published": "2026-02-07", "publisher": "arXiv", "retrieved": "2026-09-14", "source_type": "press", "title": "Frontier AI Auditing: Toward Rigorous Third-Party Assessment (arXiv 2601.11699)", "url": "https://arxiv.org/abs/2601.11699", "press_kind": "primary" }, "arxiv-goal-directedness": { "id": "arxiv-goal-directedness", "title": "Evaluating the Goal-Directedness of Large Language Models (arXiv:2504.11844)", "publisher": "arXiv", "url": "https://arxiv.org/abs/2504.11844", "published": "2025-04-16", "retrieved": "2026-09-15", "source_type": "press", "press_kind": "aggregator", "audit_status": "confirmed", "note": "Authors Tom Everitt, Cristina Garbacea, Alexis Bellot, Jonathan Richens (Google DeepMind), Henry Papadatos, Siméon Campos (SaferAI), Rohin Shah (Google DeepMind). \"Our evaluations of LLMs from Google DeepMind, OpenAI, and Anthropic show that goal-directedness is relatively consistent across tasks\". Typed press/aggregator to match the existing arxiv-frontier-auditing record; a preprint server is not an editorial outlet." }, "averi-about": { "audit_status": "confirmed", "id": "averi-about", "note": "Endorsed SB 315; co-authored AEF-1; pilots are voluntary. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026", "publisher": "AVERI", "retrieved": "2026-09-14", "source_type": "self", "title": "About", "url": "https://www.averi.org/about", "self_of": "averi" }, "averi-funding": { "audit_status": "confirmed", "id": "averi-funding", "note": "Funders: AIUC, Coefficient, Falls, Ralston, Good Forever, Fathom, Halcyon, Sympatico, non-executive lab employees; no donor majority; recusal rules; Brundage recused from auditing OpenAI two years; sold eligible shares; API credits offered by six labs. [verify 2026-09-15] reachable BUT span(s) not found: averi.10; averi.12 — needs curator review", "published": "2026", "publisher": "AVERI", "retrieved": "2026-09-14", "source_type": "self", "title": "About (funding and conflicts)", "url": "https://www.averi.org/about", "self_of": "averi" }, "averi-launch": { "audit_status": "confirmed", "id": "averi-launch", "note": "501(c)(3); founder formerly OpenAI. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026-01-15", "publisher": "Miles Brundage (Substack)", "retrieved": "2026-09-14", "source_type": "self", "title": "The Launch of AVERI", "url": "https://milesbrundage.substack.com/p/the-launch-of-averi", "self_of": "averi" }, "averi-pilot": { "audit_status": "confirmed", "id": "averi-pilot", "note": "Gemini 2.5 Flash-Lite in secure enclave with DeepMind, OpenMined, MLCommons; notes EU CoP and Illinois SB 315 requirements. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026-08", "publisher": "AVERI", "retrieved": "2026-09-14", "source_type": "self", "title": "AVERI Pilot Report: The World's First Double-Blind Evaluation of a Proprietary Language Model", "url": "https://www.averi.org/ourwork/averi-pilot-report-the-worlds-first-double-blind-eval", "self_of": "averi" }, "axios-transluce-mh": { "id": "axios-transluce-mh", "title": "Study: Chatbots are getting better at identifying suicide risk", "publisher": "Axios", "url": "https://www.axios.com/2026/08/31/chatbots-suicide-risks-identification", "published": "2026-08-31", "retrieved": "2026-09-15", "source_type": "press", "press_kind": "primary", "audit_status": "confirmed", "note": "Independent coverage of Transluce's mental-health evaluation: \"A new independent evaluation finds that leading AI models are far less likely than earlier versions to explicitly encourage suicide or reinforce delusions\" and \"the study also found most models too willing to assist with suicide-related creative writing\"; names Transluce as a San Francisco nonprofit and quotes its chief scientist. Non-self confirmed source for transluce.02." }, "cais": { "audit_status": "confirmed", "id": "cais", "note": "WMDP, HLE benchmarks; director advises xAI. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026", "publisher": "CAIS", "retrieved": "2026-09-14", "source_type": "self", "title": "Center for AI Safety", "url": "https://safe.ai/", "self_of": "cais" }, "caisi-funding": { "audit_status": "confirmed", "id": "caisi-funding", "imported_from": "kevinnbass/metr-money-figure:evaluators.csv EV12", "note": "$10M FY2026 within the NIST AI line; FY2027 request $27M. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026", "publisher": "Institute for Progress", "retrieved": "2026-09-14", "source_type": "press", "title": "Funding for CAISI", "url": "https://ifp.org/funding-for-caisi/", "press_kind": "primary" }, "caisi-usc-207": { "id": "caisi-usc-207", "title": "18 U.S. Code § 207 - Restrictions on former officers, employees, and elected officials", "publisher": "Cornell Legal Information Institute (U.S. Code)", "url": "https://www.law.cornell.edu/uscode/text/18/207", "published": "2026", "retrieved": "2026-09-15", "source_type": "filing", "audit_status": "confirmed", "note": "Federal post-employment statute: permanent, two-year and one-year restrictions on former executive-branch employees. Typed filing as a primary legal instrument hosted by a third party. Fetched 2026-09-15; span present." }, "caisi-usc-208": { "id": "caisi-usc-208", "title": "18 U.S. Code § 208 - Acts affecting a personal financial interest", "publisher": "Cornell Legal Information Institute (U.S. Code)", "url": "https://www.law.cornell.edu/uscode/text/18/208", "published": "2026", "retrieved": "2026-09-15", "source_type": "filing", "audit_status": "confirmed", "note": "Federal criminal conflict-of-interest statute binding NIST employees: bars participation in matters in which the employee has a financial interest. Typed filing as a primary legal instrument hosted by a third party. Fetched 2026-09-15; span present." }, "cnbc-caisi-fall": { "id": "cnbc-caisi-fall", "title": "Trump's head of AI safety agency CAISI resigns after months on job", "publisher": "CNBC", "url": "https://www.cnbc.com/2026/07/20/trumps-head-of-ai-safety-agency-caisi-resigns-after-months-on-job.html", "published": "2026-07-20", "retrieved": "2026-09-15", "note": "Chris Fall resigned three months after appointment; NIST director acting. [verify 2026-09-15 grok-pass] Chris Fall resigned as CAISI head after ~3 months; Raman acting.", "source_type": "press", "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "press_kind": "primary" }, "cnbc-openai-foundation": { "id": "cnbc-openai-foundation", "title": "OpenAI completes restructure, solidifying Microsoft as a major shareholder", "publisher": "CNBC", "url": "https://www.cnbc.com/2025/10/28/open-ai-for-profit-microsoft.html", "published": "2025-10-28", "retrieved": "2026-09-15", "source_type": "press", "audit_status": "confirmed", "note": "'the OpenAI Foundation will hold a 26% stake in the for-profit'; Foundation equity worth about $130 billion; nonprofit retains control. Fetched 2026-09-15; span present.", "press_kind": "primary" }, "cnbc-scale-meta": { "id": "cnbc-scale-meta", "title": "Scale AI not winding down following Meta deal, interim CEO says", "publisher": "CNBC", "url": "https://www.cnbc.com/2025/06/18/scale-ai-not-winding-down-following-meta-deal-interim-ceo-says.html", "published": "2025-06-18", "retrieved": "2026-09-15", "note": "Meta 49% stake after $14.3B investment; interim CEO Jason Droege; Wang to Meta. [verify 2026-09-15 grok-pass] Full slug .../scale-ai-not-winding-down-following-meta-deal-interim-ceo-says.html. Meta $14.3B for 49%; Droege: 'unequivocally an independent company'. [verify 2026-09-15] independently read via browser task: 'Meta has a 49% stake in Scale AI after investing $14.3 billion into the startup' (CNBC, Jun 18 2025). 49% non-voting; deal closed the prior week; founder Alexandr Wang joined Meta.", "source_type": "press", "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "press_kind": "primary" }, "coefficient-home": { "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "id": "coefficient-home", "note": "Over $7 billion in grants since 2014; Navigating Transformative AI fund with 570+ grants. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches", "published": "2026", "publisher": "Coefficient Giving", "retrieved": "2026-09-15", "source_type": "self", "title": "Coefficient Giving home", "url": "https://coefficientgiving.org/", "self_of": "coefficient" }, "coefficient-index": { "id": "coefficient-index", "title": "Coefficient Giving grants index", "publisher": "Coefficient Giving", "url": "https://coefficientgiving.org/grants/", "published": "2026-09-11", "retrieved": "2026-09-14", "note": "2,911-row snapshot; no METR-named grant; grants to ARC, RAND, FAR.AI, Redwood, Longview, Constellation, Epoch, Apollo, Irregular, Palisade.", "source_type": "index", "audit_status": "imported", "imported_from": "kevinnbass/metr-money-figure:money_flows.csv M33 and evaluators.csv" }, "coefficient-wiki": { "audit_status": "confirmed", "id": "coefficient-wiki", "note": "Formerly Open Philanthropy; board includes Moskovitz, Tuna, Karnofsky. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026", "publisher": "Wikipedia", "retrieved": "2026-09-14", "source_type": "press", "title": "Coefficient Giving", "url": "https://en.wikipedia.org/wiki/Coefficient_Giving", "press_kind": "aggregator" }, "computerworld-palisade-shutdown": { "id": "computerworld-palisade-shutdown", "title": "OpenAI's Skynet moment: Models defy human commands, actively resist orders to shut down", "publisher": "Computerworld", "url": "https://www.computerworld.com/article/3999190/openais-skynet-moment-models-defy-human-commands-actively-resist-orders-to-shut-down.html", "published": "2025-05", "retrieved": "2026-09-15", "source_type": "press", "audit_status": "confirmed", "note": "Reports Palisade's finding that o3 sabotaged the shutdown mechanism in 7 of 100 runs under explicit instruction. Fetched 2026-09-15; span present.", "press_kind": "primary" }, "csa-aisi-trends": { "audit_status": "confirmed", "id": "csa-aisi-trends", "note": "Frontier AI Bill promised, not introduced; voluntary MOUs; universal jailbreaks found; 30+ models. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches; 1 quoted span(s) found verbatim", "published": "2026-07-12", "publisher": "Cloud Security Alliance", "retrieved": "2026-09-14", "source_type": "press", "title": "UK AISI's Frontier AI Trends Report: Security Implications", "url": "https://labs.cloudsecurityalliance.org/research/csa-research-note-aisi-frontier-ai-trends-report-20260712-cs/", "press_kind": "primary" }, "csa-caisi-five-labs": { "id": "csa-caisi-five-labs", "title": "CAISI Frontier Testing Agreements Reach Five Labs", "publisher": "Cloud Security Alliance (research note)", "url": "https://labs.cloudsecurityalliance.org/research/csa-research-note-caisi-frontier-ai-testing-agreements-20260/", "published": "2026-05-05", "retrieved": "2026-09-15", "note": "Google DeepMind, Microsoft and xAI signed early-access agreements on 5 May 2026; OpenAI and Anthropic since September 2025; more than 40 evaluations completed; access includes versions with guardrails stripped back; DeepSeek V4 Pro evaluated without developer cooperation.", "source_type": "press", "audit_status": "confirmed", "press_kind": "primary" }, "csa-caisi": { "id": "csa-caisi", "title": "CAISI Frontier Testing Agreements Reach Five Labs", "publisher": "Cloud Security Alliance", "url": "https://labs.cloudsecurityalliance.org/research/csa-research-note-caisi-frontier-ai-testing-agreements-20260/", "published": "2026-05-20", "retrieved": "2026-09-14", "note": "DeepMind, Microsoft, xAI added May 2026; 40+ evaluations; TRAINS taskforce; 2025 refocus. Cited from a search snippet; not re-fetched in full.", "source_type": "press", "audit_status": "unaudited", "press_kind": "primary" }, "dealroom-grayswan": { "id": "dealroom-grayswan", "title": "Gray Swan raises $40M Series A to secure frontier AI", "publisher": "Dealroom", "url": "https://app.dealroom.co/news/note/gray-swan-raises-40m-series-a-to-secure-frontier-ai", "published": "2026-06", "retrieved": "2026-09-15", "note": "Investors: Wing, Madrona, Obvious Ventures, Snowflake Ventures, Hudson River Trading, Samsung Next, Magarac; cited in 11 recent frontier model system cards.", "source_type": "press", "audit_status": "unaudited", "found_by": "deep research pass 2026-09-15", "press_kind": "aggregator" }, "decoder-epoch": { "audit_status": "confirmed", "id": "decoder-epoch", "note": "Agreement prevented revealing support until o3 announcement. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] reachable BUT span(s) not found: epoch.01 — needs curator review [verify 2026-09-15 grok-pass] Full slug .../openai-quietly-funded-independent-math-benchmark-before-setting-record-with-o3/. Confirms OpenAI funded FrontierMath (paper thanks OpenAI); no amount.", "published": "2025-01-19", "publisher": "The Decoder", "retrieved": "2026-09-14", "source_type": "press", "title": "OpenAI quietly funded independent math benchmark", "url": "https://the-decoder.com/openai-quietly-funded-independent-math-benchmark-before-setting-record-with-o3/", "press_kind": "primary" }, "dreadnode-arxiv-airtbench": { "id": "dreadnode-arxiv-airtbench", "title": "AIRTBench: Measuring Autonomous AI Red Teaming Capabilities in Language Models", "publisher": "arXiv (Dreadnode authors)", "url": "https://arxiv.org/abs/2506.14682", "published": "2025-06-17", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Paper by Dreadnode staff (Dawson, Mulla, Landers, Caldwell); self-published research. Fetched 2026-09-15; span 'challenges from the Crucible challenge environment on the Dreadnode platform' present. Reports results on Claude 3.7 Sonnet, Gemini 2.5 Pro, GPT-4.5 and others: a published evaluation of frontier models.", "self_of": "dreadnode" }, "dreadnode-github-airtbench": { "id": "dreadnode-github-airtbench", "title": "dreadnode/AIRTBench-Code", "publisher": "GitHub (Dreadnode)", "url": "https://github.com/dreadnode/AIRTBench-Code", "published": "2025-06", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Code repository for AIRTBench, Apache-2.0 badge; fetched 2026-09-15 (page title 'Code Repository for: AIRTBench: Measuring Autonomous AI Red Teaming Capabilities in Language Models'). Supports M.1 (open code) for the organization's own benchmark.", "self_of": "dreadnode" }, "dreadnode-google-gemini3": { "id": "dreadnode-google-gemini3", "title": "A new era of intelligence with Gemini 3", "publisher": "Google", "url": "https://blog.google/products/gemini/gemini-3/", "published": "2025-11-18", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Lab-published launch post (third-party primary document, typed self per repo precedent for lab pages such as openai-kolter-board and gdm-gemini-testing). Fetched 2026-09-15; span 'obtained independent assessments from industry experts like Apollo, Vaultis, Dreadnode and more' present. This is the document the existing gdm-gemini-testing record describes; that record's URL points at the homepage and should be retargeted here or dropped.", "self_of": "google" }, "dreadnode-securityweek": { "id": "dreadnode-securityweek", "title": "Offensive AI Startup Dreadnode Secures $14M to Stress-Test AI Systems", "publisher": "SecurityWeek", "url": "https://www.securityweek.com/offensive-ai-startup-dreadnode-secures-14m-to-stress-test-ai-systems/", "published": "2025-02-25", "retrieved": "2026-09-15", "source_type": "press", "audit_status": "confirmed", "note": "Fetched 2026-09-15; span 'two new products — Strikes and Spyglass — that form the core of its platform' present. Non-self corroboration of the round and products.", "press_kind": "primary" }, "dreadnode-seriesa": { "id": "dreadnode-seriesa", "title": "Dreadnode Secures $14M to Build AI Systems that Advance the State of Offensive Security", "publisher": "Dreadnode", "url": "https://dreadnode.io/company/newsroom/series-a/", "published": "2025-02-25", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Fetched 2026-09-15; spans present: 'announced its $14 million Series A funding round, led by Decibel'; 'The company reveals two new advanced offensive AI solutions—Strikes and Spyglass'. Own-page admission of venture backing and of the product line.", "self_of": "dreadnode" }, "dreadnode-time-gdm-pledge": { "id": "dreadnode-time-gdm-pledge", "title": "Exclusive: 60 U.K. Lawmakers Accuse Google of Breaking AI Safety Pledge", "publisher": "Time", "url": "https://time.com/7313320/google-deepmind-gemini-ai-safety-pledge/", "published": "2025-08-29", "retrieved": "2026-09-15", "source_type": "press", "audit_status": "confirmed", "note": "Fetched 2026-09-15; span 'a “diverse group of external experts,” including Apollo Research, Dreadnode, and Vaultis' present (DeepMind spokesperson on Gemini 2.5 Pro sharing). Independent editorial corroboration of the Dreadnode engagement.", "press_kind": "primary" }, "epoch-clarify": { "audit_status": "confirmed", "id": "epoch-clarify", "note": "OpenAI commissioned 300 problems, owns them, has statements and solutions except 50-problem holdout; contractor communication gap. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches; 1 quoted span(s) found verbatim", "published": "2025-01-23", "publisher": "Epoch AI", "retrieved": "2026-09-14", "source_type": "self", "title": "Clarifying the creation and use of the FrontierMath benchmark", "url": "https://epoch.ai/latest/openai-and-frontiermath", "self_of": "epoch" }, "epoch-gpt54-frontiermath": { "id": "epoch-gpt54-frontiermath", "title": "GPT-5.4 set a new record on FrontierMath", "publisher": "Epoch AI (Substack)", "url": "https://epochai.substack.com/p/gpt-54-set-a-new-record-on-frontiermath", "published": "2026-03-05", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "States 'We had pre-release access to evaluate the model' and that FrontierMath was funded by OpenAI. Fetched 2026-09-15; span present.", "self_of": "epoch" }, "epoch-musespark-x": { "id": "epoch-musespark-x", "title": "Epoch AI on X: pre-release access to Meta's Muse Spark", "publisher": "Epoch AI (X)", "url": "https://x.com/EpochAIResearch/status/2041947954202988757", "published": "2026", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "'We had pre-release access to Meta’s new Muse Spark model and evaluated it on FrontierMath.' Fetched 2026-09-15; span present.", "self_of": "epoch" }, "epoch-transparency-page": { "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "id": "epoch-transparency-page", "note": "Re-read in full 2026-09-15. Funders above $70K: Coefficient (8 grants, $25,113,611 total), SFF $195,000 (May 2026), Girish Sastry $100,000, Tallinn $600,000 DAF, Aschenbrenner $200,000 via Manifund, Govindaiah $400,000 DAF, Schmidt Sciences (FrontierMath Open Problems pilot), Sentinel Bio $165,000, Shulman $100,000. Clients: Bridgewater AIA Labs, EU AI Office, Google DeepMind (2024-2026), OpenAI (2024-2026), ARIA, AI Index, Blitzy, EPRI, METR, Sequoia Capital Global Equities, UK DSIT, xAI (2025), Anthropic (2024 pilot). States it invests part of its funds in semiconductor and AI stocks and charges at least industry-consultant rates.", "published": "2026", "publisher": "Epoch AI", "retrieved": "2026-09-15", "source_type": "self", "title": "Transparency", "url": "https://epoch.ai/about/transparency", "self_of": "epoch" }, "epoch-transparency": { "audit_status": "confirmed", "id": "epoch-transparency", "imported_from": "kevinnbass/metr-money-figure:evaluators.csv EV03", "note": "Donations above $70K listed; Google DeepMind and OpenAI as paying consultation clients; Tallinn DAF $600K. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026", "publisher": "Epoch AI", "retrieved": "2026-09-14", "source_type": "self", "title": "Transparency", "url": "https://epoch.ai/about/transparency", "self_of": "epoch" }, "equistamp-site": { "id": "equistamp-site", "title": "EquiStamp - Research Operations for AI Safety", "publisher": "EquiStamp", "url": "https://equistamp.com/", "published": "2026", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Company site: evaluation implementation, data annotation, red/blue teaming for AI safety organizations; \"in the process of becoming a Public Benefit Corporation\"; clients and partners list. No conflict-of-interest policy on the page. Fetched 2026-09-15; spans present.", "self_of": "equistamp" }, "equistamp-substack": { "id": "equistamp-substack", "title": "Isolating \"AI Safety\" to an ideological niche will backfire against fundamental rights", "publisher": "Stress Testing Reality (Substack, Katalina Hernández)", "url": "https://stresstestingreality.substack.com/p/isolating-ai-safety-to-an-ideological", "published": "2026", "retrieved": "2026-09-15", "source_type": "press", "audit_status": "confirmed", "note": "Names EquiStamp, METR, Epoch AI, FAR.AI, SaferAI, SecureBio, Clarity AI Research and CeSIA as winners of Lots 1, 3 and 4 of the EUR 9 million AI Office tender. Author discloses an ongoing relationship with EquiStamp (head of legal), so not independent of the company. Newsletter, typed press aggregator. Fetched 2026-09-15; span present.", "press_kind": "aggregator" }, "euaio-ai-act-92": { "id": "euaio-ai-act-92", "title": "Article 92: Power to conduct evaluations (AI Act Service Desk)", "publisher": "European Commission (AI Act Service Desk)", "url": "https://ai-act-service-desk.ec.europa.eu/en/ai-act/article-92", "published": "2024-07-12", "retrieved": "2026-09-15", "source_type": "filing", "audit_status": "confirmed", "note": "Regulation (EU) 2024/1689 Article 92 text on the Commission's official service desk: the AI Office may conduct evaluations and request access through APIs or other technical means including source code; providers must comply or face fines. Typed filing as a primary legal instrument. Fetched 2026-09-15; span present." }, "euaio-ai-office-page": { "id": "euaio-ai-office-page", "title": "European AI Office", "publisher": "European Commission (DG CONNECT)", "url": "https://digital-strategy.ec.europa.eu/en/policies/ai-office", "published": "2026", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Office description: established within the Commission; enforces GPAI rules; powers include conducting evaluations, requesting information and measures, applying sanctions. Fetched 2026-09-15; spans present.", "self_of": "euaio" }, "euaio-ceo-staffregs": { "id": "euaio-ceo-staffregs", "title": "New EU Staff Regulations adopted: Small steps on revolving door, giant leaps still needed", "publisher": "Corporate Europe Observatory", "url": "https://corporateeurope.org/en/revolving-doors/2013/07/new-eu-staff-regulations-adopted-small-steps-revolving-door-giant-leaps", "published": "2013-07", "retrieved": "2026-09-15", "source_type": "press", "audit_status": "confirmed", "note": "Describes Staff Regulations Article 11 (conflict-of-interest screening) and Article 16 (two-year post-employment notification; one-year lobbying cooling-off for senior officials). NGO analysis, typed press aggregator. Fetched 2026-09-15; spans present.", "press_kind": "aggregator" }, "euaio-cop-text": { "id": "euaio-cop-text", "title": "General-Purpose AI Code of Practice: Safety and Security chapter (mirror)", "publisher": "code-of-practice.ai (Alexander Zacherl mirror of the Commission text)", "url": "https://code-of-practice.ai/?section=safety-security", "published": "2025-07", "retrieved": "2026-09-15", "source_type": "press", "audit_status": "confirmed", "note": "Verbatim mirror of the Code's Safety and Security chapter: Commitment 7 Safety and Security Model Reports submitted to the AI Office; Measure on independent external evaluators. Third-party mirror, typed press aggregator; the Commission PDF is the authoritative text. Fetched 2026-09-15; span present.", "press_kind": "aggregator" }, "euaio-newsletter-77": { "id": "euaio-newsletter-77", "title": "The EU AI Act Newsletter #77: AI Office Tender", "publisher": "EU AI Act Newsletter (Substack)", "url": "https://artificialintelligenceact.substack.com/p/the-eu-ai-act-newsletter-77-ai-office", "published": "2025-05", "retrieved": "2026-09-15", "source_type": "press", "audit_status": "confirmed", "note": "States the EUR 9,080,000 technical-assistance tender is divided into six lots under Articles 89, 92 and 93 of the AI Act. Newsletter, typed press aggregator. Fetched 2026-09-15; span present.", "press_kind": "aggregator" }, "euaio-office-decision": { "id": "euaio-office-decision", "title": "Commission Decision Establishing the European AI Office", "publisher": "European Commission (DG CONNECT)", "url": "https://digital-strategy.ec.europa.eu/en/library/commission-decision-establishing-european-ai-office", "published": "2024-01-24", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "The Office is established within the Commission in the administrative structure of DG CONNECT, subject to its annual management plan. Fetched 2026-09-15; span present.", "self_of": "euaio" }, "euaio-staff-regs": { "id": "euaio-staff-regs", "title": "Staff Regulations of Officials of the European Union (consolidated text)", "publisher": "EUR-Lex", "url": "https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX:01962R0031-20240101", "published": "2024-01-01", "retrieved": "2026-09-15", "source_type": "filing", "audit_status": "unaudited", "note": "Regulation No 31 (EEC), 11 (EAEC) consolidated: Article 11a (conflict of interest), Article 16 (activities after leaving the service). EUR-Lex returns an empty body to the CI normalizer and to curl; not verified by fetch. Cited for the primary text; the CEO article carries the confirmed spans." }, "euaio-tender-call": { "id": "euaio-tender-call", "title": "Forthcoming call for tenders: Artificial Intelligence Act - Technical Assistance for AI Safety", "publisher": "European Commission (DG CONNECT)", "url": "https://digital-strategy.ec.europa.eu/en/news/forthcoming-call-tenders-artificial-intelligence-act-technical-assistance-ai-safety", "published": "2025-04-28", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Commission announcement: tender split into six lots to support monitoring of compliance and assessment of GPAI systemic risks. Same record as in leads/bounds/euaio.json. Fetched 2026-09-15; span present.", "self_of": "euaio" }, "evaluators-ledger": { "id": "evaluators-ledger", "title": "evaluators.csv (24 evaluators: Coefficient and SFF totals, lab money, government contracts, leadership)", "publisher": "Kevin Bass (GitHub)", "url": "https://raw.githubusercontent.com/kevinnbass/metr-money-figure/master/research/evaluators.csv", "published": "2026-09-14", "retrieved": "2026-09-15", "note": "Parallel scorecard with index totals per evaluator; imported as leads. Repository default branch is master (the earlier main path returned 404). Sibling files: coefficient_pages.csv, tallinn-donations-ledger-2026-09-13.csv, vanguard-fy*-schedule-i.csv, finances.csv. [audit 2026-09-15] URL repointed from the GitHub blob page (JavaScript-rendered) to the raw file so quoted spans can be checked; status set to imported, which is what it is (a row-level lead copied from another project, supporting a 2 and nothing else until re-derived).", "source_type": "ledger", "audit_status": "imported", "imported_from": "kevinnbass/metr-money-figure:evaluators.csv" }, "far-30m": { "id": "far-30m", "title": "FAR.AI Secures Over $30 Million in Multi-Funder Support", "publisher": "FAR.AI", "url": "https://www.far.ai/blog/30m-multi-funder-support", "published": "2026-07-16", "retrieved": "2026-09-14", "note": "Funders: Coefficient Giving, Schmidt Sciences, SFF, CSET, AI Safety Fund (Frontier Model Forum). Cited from a search snippet; not re-fetched in full. [verify 2026-09-15 grok-pass] Live at /blog/ (not /news/). 'over $30M in funding commitments throughout 2025' from Coefficient, Schmidt, SFF, CSET, AISF — 2025 commitments, not a cumulative.", "source_type": "self", "audit_status": "confirmed", "self_of": "farai" }, "far-transparency": { "audit_status": "confirmed", "id": "far-transparency", "imported_from": "kevinnbass/metr-money-figure:evaluators.csv EV09", "note": "Revenue from for-profit AI developers capped at 10% of annual revenue; market rates; right to publish retained; 990 TY2024 earned revenue $429K of $24.3M. [verify 2026-09-15] fetched; page reachable, citation matches; 1 quoted span(s) found verbatim", "published": "2026", "publisher": "FAR.AI", "retrieved": "2026-09-14", "source_type": "self", "title": "Transparency", "url": "https://www.far.ai/transparency", "self_of": "farai" }, "forbes-grayswan-2024": { "id": "forbes-grayswan-2024", "title": "Gray Swan AI Is Working With OpenAI to Red Team Its Models", "publisher": "Forbes", "url": "https://www.forbes.com/sites/sarahemerson/2024/10/29/this-hacker-team-is-bulletproofing-ai-models-for-companies-like-openai/", "published": "2024-10-29", "retrieved": "2026-09-14", "note": "Kolter recused from Gray Swan-OpenAI interactions; $5.5M seed. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15 grok-pass] Seed $5.5M; clients OpenAI, Anthropic, UK AISI. Partial/paywall.", "source_type": "press", "audit_status": "confirmed", "press_kind": "primary", "archive": "https://web.archive.org/web/2025id_/https://www.forbes.com/sites/sarahemerson/2024/10/29/this-hacker-team-is-bulletproofing-ai-models-for-companies-like-openai/" }, "forbes-grayswan-2026": { "id": "forbes-grayswan-2026", "title": "This AI Startup's Army of 15,000 Hackers", "publisher": "Forbes", "url": "https://www.forbes.com/sites/rashishrivastava/2026/05/28/this-ai-startups-army-of-15000-hackers-pressure-test-claude-gpt-5-and-gemini/", "published": "2026-05-28", "retrieved": "2026-09-14", "note": "$200M valuation; clients OpenAI, Anthropic, Google DeepMind, Meta, xAI, ByteDance. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15 grok-pass] Series A $40M, $200M valuation; 15k arena users; labs + Snowflake.", "source_type": "press", "audit_status": "confirmed", "press_kind": "primary" }, "fortune-averi": { "id": "fortune-averi", "title": "Former OpenAI policy chief debuts new institute called AVERI", "publisher": "Fortune", "url": "https://fortune.com/2026/01/15/former-openai-policy-chief-creates-nonprofit-institute-calls-for-independent-safety-audits-of-frontier-ai-models/", "published": "2026-01-15", "retrieved": "2026-09-15", "note": "$7.5M raised toward $13M; funders incl. AIUC, Coefficient, Halcyon, Fathom; donations from non-executive lab employees. [verify 2026-09-15 grok-pass] AVERI raised $7.5M toward $13M; funders match AVERI about; 'will not conduct audits itself' is dated (they now do audits) — date the claim.", "source_type": "press", "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "press_kind": "primary" }, "fortune-hf-nvidia": { "id": "fortune-hf-nvidia", "title": "Hugging Face goes from a scrappy startup to $13 billion Nvidia acquisition", "publisher": "Fortune", "url": "https://fortune.com/2026/09/03/hugging-face-goes-from-a-scrappy-startup-named-after-an-emoji-to-13-billion-nvidia-acquisition/", "published": "2026-09-03", "retrieved": "2026-09-15", "note": "Nvidia agreed to acquire Hugging Face for $12,930,300,000. [verify 2026-09-15 grok-pass] 'Nvidia has agreed to acquire Hugging Face for $12,930,300,000' (3 Sep 2026).", "source_type": "press", "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "press_kind": "primary" }, "founderspledge-frontier": { "id": "founderspledge-frontier", "title": "Frontier AI grantmaking", "publisher": "Founders Pledge", "url": "https://www.founderspledge.com/research/frontier-ai-grantmaking", "published": "2026", "retrieved": "2026-09-15", "note": "$500,000 to FAR.AI Integrity Team for EU AI Office manipulation evaluations. [verify 2026-09-15 grok-pass] Live; GCR fund 'over $1M' H1 2026 — no METR $184k; cannot confirm T15.", "source_type": "self", "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "self_of": "founders-pledge-frontier" }, "gdm-gemini-testing": { "audit_status": "unverifiable", "id": "gdm-gemini-testing", "note": "Dreadnode and Vaultis named as external experts (cyber, CBRN, extremism). Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches [audit 2026-09-15] the URL resolves to the deepmind.google homepage, which does not mention Dreadnode; the Gemini 3 launch post and press are cited instead. Supports nothing.", "published": "2025", "publisher": "Google DeepMind", "retrieved": "2026-09-14", "source_type": "self", "title": "Gemini model documentation, external testing", "url": "https://deepmind.google/", "self_of": "google" }, "grayswan-forbesau-2024": { "id": "grayswan-forbesau-2024", "title": "This hacker team is bulletproofing AI models for companies like OpenAI and Anthropic", "publisher": "Forbes Australia", "url": "https://www.forbes.com.au/news/innovation/this-hacker-team-is-bulletproofing-ai-models-for-companies-like-openai-and-anthropic/", "published": "2024-11-07", "retrieved": "2026-09-15", "source_type": "press", "audit_status": "confirmed", "note": "Syndicated copy of the Forbes (29 Oct 2024) article; fetched 2026-09-15 with fetch_text; span 'he has recused himself from interactions between the two companies' present verbatim. Fetchable where forbes.com is not.", "press_kind": "primary" }, "grayswan-latentspace-2026": { "id": "grayswan-latentspace-2026", "title": "Red-Teaming after Mythos — Zico Kolter & Matt Fredrikson, Gray Swan", "publisher": "Latent Space", "url": "https://www.latent.space/p/gray-swan", "published": "2026-06-22", "retrieved": "2026-09-15", "source_type": "press", "audit_status": "confirmed", "note": "Podcast/newsletter page; fetched 2026-09-15; span 'the adversarial red teaming tool that Anthropic used to evaluate the robustness of their models' present (Shade). Lead for X.1: a lab using the evaluator's product.", "press_kind": "aggregator" }, "grayswan-seriesa": { "audit_status": "confirmed", "id": "grayswan-seriesa", "note": "$40M; Wing and Madrona; cited in 11 system cards; Cygnal, Shade, Arena products. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches; 1 quoted span(s) found verbatim", "published": "2026-05-28", "publisher": "Gray Swan", "retrieved": "2026-09-14", "source_type": "self", "title": "Gray Swan announces Series A", "url": "https://www.grayswan.ai/news/gray-swan-announces-series-a", "self_of": "grayswan" }, "grayswan-snowflake-anthropic": { "id": "grayswan-snowflake-anthropic", "title": "Snowflake and Anthropic Announce $200 Million Partnership to Bring Agentic AI to Global Enterprises", "publisher": "Snowflake", "url": "https://www.snowflake.com/en/news/press-releases/snowflake-and-anthropic-announce-200-million-partnership-to-bring-agentic-ai-to-global-enterprises/", "published": "2025-12-03", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Third-party primary document (company press release), typed self per repo precedent for lab/investor pages. Fetched 2026-09-15; states a 'multi-year, $200 million agreement' making Claude available in Snowflake, a commercial commitment, not an equity stake. Closes the open question on Snowflake Ventures as a lab investor: no equity documented.", "self_of": "snowflake-ventures" }, "hal": { "audit_status": "confirmed", "id": "hal", "note": "Open agent leaderboard; AEF founding member. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026", "publisher": "Princeton", "retrieved": "2026-09-14", "source_type": "self", "title": "Holistic Agent Leaderboard", "url": "https://hal.cs.princeton.edu/", "self_of": "hal" }, "hfoai-hf-pricing": { "id": "hfoai-hf-pricing", "title": "Hugging Face – Pricing", "publisher": "Hugging Face", "url": "https://huggingface.co/pricing", "published": "2026", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Own page; fetched 2026-09-15; span 'Instant setup for growing teams $20 /month per user' present (Team plan), with PRO and Enterprise tiers. Documents commercial products for X.", "self_of": "hfoai" }, "humane-board": { "id": "humane-board", "title": "Board & Advisory - Humane Intelligence", "publisher": "Humane Intelligence", "url": "https://www.humane-intelligence.org/board-advisory/", "published": "2026-07", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Board of directors and advisory group with bios: board president previously on OpenAI's Human Data team; advisory-group member is a Senior Research Scientist at Meta. Fetched 2026-09-15; spans present.", "self_of": "humane" }, "humane-int": { "audit_status": "confirmed", "id": "humane-int", "note": "Public red-teaming with NIST and IMDA. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches", "published": "2026", "publisher": "Humane Intelligence", "retrieved": "2026-09-14", "source_type": "self", "title": "Humane Intelligence", "url": "https://www.humane-intelligence.org/", "self_of": "humane" }, "ifp-caisi-funding": { "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "id": "ifp-caisi-funding", "note": "CAISI about $15M FY2026 ($10M appropriated plus a TMF loan); FY2027 request $27M; equipped estimate about $84M per year. [verify 2026-09-15] fetched 2026-09-15; page reachable BUT quoted span(s) not found: caisi.11 — needs curator review [verify 2026-09-15] reachable BUT span(s) not found: caisi.11 — needs curator review [verify 2026-09-15 grok-pass] CAISI ~$15M now ($10M FY26 + $10M TMF loan); needs ~$84M equipped / $26M limited; FY27 PBR $27M.", "published": "2026", "publisher": "Institute for Progress", "retrieved": "2026-09-15", "source_type": "press", "title": "What Will It Cost for the US to Be Ready for the Next Big AI Breakthrough?", "url": "https://ifp.org/funding-for-caisi/", "press_kind": "primary" }, "instrumentl-arc-990": { "audit_status": "unaudited", "found_by": "deep research pass 2026-09-15", "id": "instrumentl-arc-990", "note": "Reports ARC contributed $4,553,935 in grants during 2024 (single grant); secondary extraction of the e-filed 990. [verify 2026-09-15] reachable BUT span(s) not found: metr.21 — needs curator review", "published": "2025", "publisher": "Instrumentl", "retrieved": "2026-09-15", "source_type": "ledger", "title": "Alignment Research Center 990 report", "url": "https://www.instrumentl.com/990-report/alignment-research-center" }, "irregular-xai-series-b": { "id": "irregular-xai-series-b", "title": "Series B funding round", "publisher": "xAI", "url": "https://x.ai/news/series-b", "published": "2024-05-26", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Third-party primary document (a frontier developer's own funding post), typed self per repo precedent. Fetched 2026-09-15; span 'with participation from key investors including Valor Equity Partners, Vy Capital, Andreessen Horowitz, Sequoia Capital' present. Establishes Sequoia Capital as an investor in a frontier developer for F.4/G.3.", "self_of": "xai" }, "itbb-watchdogs": { "audit_status": "confirmed", "id": "itbb-watchdogs", "note": "Funding pipeline analysis; METR diversification; Redwood revenue volatility; ~$25M Open Phil to Redwood historically. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches", "published": "2026-06-03", "publisher": "Inside The Black Box (Substack)", "retrieved": "2026-09-14", "source_type": "press", "title": "Who Funds the AI Safety Watchdogs", "url": "https://itbb.substack.com/p/who-funds-the-watchdogs", "press_kind": "aggregator" }, "kolter-bio": { "audit_status": "confirmed", "id": "kolter-bio", "note": "Chairs OpenAI SSC; co-founder and Chief Scientist of Gray Swan. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches; 1 quoted span(s) found verbatim", "published": "2026", "publisher": "Zico Kolter", "retrieved": "2026-09-14", "source_type": "self", "title": "Bio", "url": "http://zkolter.github.io/bio/", "self_of": "grayswan" }, "lastexam-hle": { "id": "lastexam-hle", "title": "Humanity's Last Exam", "publisher": "Center for AI Safety and Scale AI", "url": "https://lastexam.ai/", "published": "2025", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Author affiliations '1 Center for AI Safety, 2 Scale AI'; Nature citation lists CAIS, Scale AI and the contributors consortium as authors. Fetched 2026-09-15; span present.", "self_of": "cais" }, "longtermwiki-audit": { "audit_status": "confirmed", "id": "longtermwiki-audit", "note": "Scale AI SEAL first third-party evaluator authorized by US AISI; ecosystem overview. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026-01-29", "publisher": "Longterm Wiki", "retrieved": "2026-09-14", "source_type": "press", "title": "Third-Party Model Auditing", "url": "https://www.longtermwiki.com/wiki/E450", "press_kind": "aggregator" }, "longtermwiki-far-cap": { "id": "longtermwiki-far-cap", "title": "FAR AI", "publisher": "Longterm Wiki", "url": "https://www.longtermwiki.com/wiki/E138", "published": "2026", "retrieved": "2026-09-15", "note": "States FAR AI caps revenue from for-profit AI developers at 10% of total annual revenue; FY2024 990 revenue $24.3M.", "source_type": "ledger", "audit_status": "unaudited", "found_by": "deep research pass 2026-09-15" }, "longtermwiki-far": { "id": "longtermwiki-far", "title": "FAR AI", "publisher": "Longterm Wiki", "url": "https://www.longtermwiki.com/wiki/E138", "published": "2026-02-26", "retrieved": "2026-09-14", "note": "EU AI Office tender EC-CNECT/2025/OP/0032, Lot 1 led by FAR.AI with SecureBio and SaferAI; FY2024 revenue. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15 grok-pass] Loads. Coefficient components ~$28.675M, $6.65M, $2.16M, $1.7M + $12M regrant; FY2024 rev $24.3M; dev-revenue cap 10%; 2025 commitments >$30M. Tier-2 lead for T03 components, not a filing.", "source_type": "press", "audit_status": "confirmed", "press_kind": "aggregator" }, "madrona-grayswan": { "id": "madrona-grayswan", "title": "Why We Are Co-Leading Gray Swan's Series A", "publisher": "Madrona", "url": "https://www.madrona.com/gray-swan-series-a/", "published": "2026-05-28", "retrieved": "2026-09-15", "note": "Co-lead's account of the round; notes Kolter serves on the board of OpenAI. [verify 2026-09-15 grok-pass] $40M Series A co-led with Wing; Obvious, Snowflake Ventures, HRT, Samsung Next, Magarac.", "source_type": "press", "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "press_kind": "primary" }, "marktechpost-pace": { "audit_status": "confirmed", "id": "marktechpost-pace", "note": "Investigators took no payment; ~$400K API credits; embedded evaluator terms: desks, badges, publication without editorial control. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026-09-13", "publisher": "MarkTechPost", "retrieved": "2026-09-14", "source_type": "press", "title": "Anthropic's 3-Step 'Pace the Frontier' Plan", "url": "https://www.marktechpost.com/2026/09/13/anthropics-3-step-pace-the-frontier-plan-wins-openai-xai-and-microsoft-support-is-it-too-late-to-slow-ai-down/", "press_kind": "aggregator" }, "metr-990-fy2024": { "id": "metr-990-fy2024", "title": "METR FY2024 Form 990 (EIN 99-1219864)", "publisher": "ProPublica Nonprofit Explorer", "url": "https://projects.propublica.org/nonprofits/organizations/991219864", "published": "2025", "retrieved": "2026-09-14", "note": "Officers and directors, related organization ARC, $4.5M transfer from ARC, revenue $13.6M. ProPublica summary re-fetched 2026-09-14: revenue $13,639,155, contributions $13,603,035. [verify 2026-09-15 grok-pass] Org card FY2024 revenue $13,639,155; no Schedule I in HTML — grant-by-grant needs the PDF.", "source_type": "filing", "audit_status": "confirmed", "imported_from": "kevinnbass/metr-money-figure:finances.csv N01-N22" }, "metr-about": { "id": "metr-about", "title": "About METR", "publisher": "METR", "url": "https://metr.org/about", "published": "2026-08", "retrieved": "2026-09-14", "note": "States METR has not accepted funding from AI companies; notes reliance on free tokens.", "source_type": "self", "audit_status": "confirmed", "self_of": "metr" }, "metr-audacious-2024": { "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "id": "metr-audacious-2024", "note": "Approximately $17 million of the ~$38M Canary commitment supports work at METR. [verify 2026-09-15] fetched; page reachable, citation matches; 1 quoted span(s) found verbatim", "published": "2024-10-09", "publisher": "METR", "retrieved": "2026-09-15", "source_type": "self", "title": "New Support Through The Audacious Project", "url": "https://metr.org/blog/2024-10-09-new-support-through-the-audacious-project/", "self_of": "metr" }, "metr-coi-policy": { "id": "metr-coi-policy", "title": "Conflict of interest policy (version 1.0)", "publisher": "METR", "url": "https://metr.org/coi-policy.pdf", "published": "2026-08-28", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "PDF (172 KB, about 17k characters via pdftotext) linked from metr.org/about as \"Our current conflict of interest policy\". Staff disclosure form, three conflict tiers with external-disclosure requirements, provisional recusal, ban on direct equity or debt in frontier AI companies, board ineligibility for frontier-company employees, declining donations made by or at the direction of frontier AI companies or their employees, free compute credits acceptable, and \"To date, we have not received payment for work on company-identifying risk assessments\". No cooling-off period. fetch_text cannot extract PDF text; spans verified against pdftotext output on 2026-09-15.", "self_of": "metr" }, "metr-donor-rule-history": { "id": "metr-donor-rule-history", "title": "METR donor rule wording over time (Wayback captures of metr.org/about and /donate)", "publisher": "Internet Archive", "url": "https://web.archive.org/web/2025*/metr.org/about", "published": "2025-08", "retrieved": "2026-09-14", "note": "No lab-money rule on the April 2024 page; a footnote about compensation for evaluations appears April 2025; 'has not accepted funding from AI companies' plus free credits language from August 2025; Sept 2025 footnote says open to donations from individual lab employees. Date range of captures 2024-2026; published set to 2025-08 (first capture with the current rule wording). [verify 2026-09-15 grok-pass] Wayback wildcard 503s; use concrete snapshots (/web/202504.../, /web/202508.../) for history. Current wording is on metr.org/about.", "source_type": "ledger", "audit_status": "unverifiable", "imported_from": "kevinnbass/metr-money-figure:donor_rule.csv DR01-DR12" }, "metr-funding-2026": { "audit_status": "confirmed", "id": "metr-funding-2026", "note": "Independence framing; growth in philanthropic commitments. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches; 1 quoted span(s) found verbatim", "published": "2026-08-14", "publisher": "METR", "retrieved": "2026-09-14", "source_type": "self", "title": "Funding update", "url": "https://metr.org/blog/2026-08-14-funding-update/", "self_of": "metr" }, "metr-hcast-public": { "id": "metr-hcast-public", "title": "METR/hcast-public (GitHub README)", "publisher": "METR (GitHub)", "url": "https://github.com/METR/hcast-public", "published": "2025", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "\"This repo contains source code for a subset of the tasks used in the HCAST: Human-Calibrated Autonomy Software Tasks paper\"; tasks conform to the METR Task Standard. Supports anchor 3 (tasks partly open); the rest of the suite is withheld, and no third-party re-run is documented, so M.1 does not reach 4.", "self_of": "metr" }, "metr-hf-investigation": { "audit_status": "confirmed", "id": "metr-hf-investigation", "note": "Terms of engagement: six days on premises, OpenAI could redact non-public info, gave feedback on structure and tone; redaction summary statement. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026-08-26", "publisher": "METR", "retrieved": "2026-09-14", "source_type": "self", "title": "Brief independent investigation of agents' behavior in the OpenAI / Hugging Face hacking incident", "url": "https://metr.org/blog/2026-08-26-openai-hugging-face-incident-investigation/", "self_of": "metr" }, "metr-money-figure": { "audit_status": "confirmed", "capture": "git clone --depth 1, commit f64df65a16400fd88a77536111851695acc9a928", "discovered_via": "https://x.com/kevinnbass/status/2099621874279817638", "id": "metr-money-figure", "note": "Third-party investigative ledger: 150 money-flow rows, 990 e-file object ids, SFF and Tallinn ledgers, DAF Schedule I rows, Wayback captures, four internal audits with per-row verdicts. Its own audit flags overclaims in the rendered figure's legend and 'Direct: $0' wording. Treated here as leads, not findings. Cloned at commit f64df65a1640 on 2026-09-14. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026-09-14", "publisher": "Kevin Bass (GitHub)", "retrieved": "2026-09-15", "source_type": "ledger", "title": "metr-money-figure research ledger and audits", "url": "https://github.com/kevinnbass/metr-money-figure/tree/master" }, "metr-regs": { "id": "metr-regs", "title": "Frontier AI safety regulations: A reference for lab staff", "publisher": "METR", "url": "https://metr.org/notes/2026-01-29-frontier-ai-safety-regulations/", "published": "2026-01-29", "retrieved": "2026-09-14", "note": "EU CoP: adequate access incl. helpful-only versions, 20 business days, external evaluators each new frontier model; enforceable Aug 2026. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15 grok-pass] SB 315: independent auditor, no financial interest, pay not tied to findings; audits from 2028.", "source_type": "self", "audit_status": "confirmed", "self_of": "metr" }, "metr-team": { "id": "metr-team", "title": "Team", "publisher": "METR", "url": "https://metr.org/team/", "published": "2026", "retrieved": "2026-09-14", "note": "Board and advisors listed on the team page: Gleave (FAR.AI), Dattani (AIUC), Radford, Mascorro (a16z), Bengio; former advisors Karnofsky, Christiano appear in earlier captures. Re-fetched 2026-09-14: current advisors Gleave (board), Radford, Mascorro, Dattani (board), Bengio.", "source_type": "self", "audit_status": "confirmed", "imported_from": "kevinnbass/metr-money-figure:board.csv B05-B13", "self_of": "metr" }, "mlcommons-ailuminate": { "id": "mlcommons-ailuminate", "title": "AILuminate", "publisher": "MLCommons", "url": "https://mlcommons.org/ailuminate/", "published": "2026-04-27", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Benchmark page: \"MLCommons is a 501c6 non-profit organization\"; sponsors; AI Risk & Reliability working-group contributor list naming Amazon, Anthropic, Cohere, Google, Google DeepMind, Meta FAIR, Microsoft, OpenAI. Fetched 2026-09-15; spans present.", "self_of": "mlcommons" }, "mlcommons-demo-dataset": { "id": "mlcommons-demo-dataset", "title": "MLCommons Releases AILuminate Creative Commons DEMO Benchmark Prompt Dataset to Github", "publisher": "MLCommons", "url": "https://mlcommons.org/2025/01/ailuminate-demo-benchmark-prompt-dataset/", "published": "2025-01-15", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "1,200-prompt DEMO set public; Practice set on request; Official Test dataset private to prevent gaming; public reports of 13 systems-under-test. Fetched 2026-09-15; spans present.", "self_of": "mlcommons" }, "mlcommons-leadership": { "id": "mlcommons-leadership", "title": "Leadership", "publisher": "MLCommons", "url": "https://mlcommons.org/about-us/leadership/", "published": "2026", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Board and officers: \"Peter Mattson is a senior staff engineer at Google. He founded and is President of MLCommons\". Fetched 2026-09-15; span present.", "self_of": "mlcommons" }, "mlcommons-propublica": { "id": "mlcommons-propublica", "title": "Mlcommons Association - Nonprofit Explorer", "publisher": "ProPublica", "url": "https://projects.propublica.org/nonprofits/organizations/850546914", "published": "2026", "retrieved": "2026-09-15", "source_type": "index", "audit_status": "confirmed", "note": "EIN 85-0546914; designated 501(c)(6) business league; net assets $4,551,326 at latest fiscal year end; revenue fields blank in the summary. Fetched 2026-09-15." }, "mlcommons-siliconangle": { "id": "mlcommons-siliconangle", "title": "MLCommons releases new AILuminate benchmark for measuring AI model safety", "publisher": "SiliconANGLE", "url": "https://siliconangle.com/2024/12/04/mlcommons-releases-new-ailuminate-benchmark-measuring-llm-safety/", "published": "2024-12-04", "retrieved": "2026-09-15", "source_type": "press", "audit_status": "confirmed", "note": "\"MLCommons is an industry consortium backed by several dozen tech firms\"; working group included employees of Nvidia, Intel, Qualcomm. Named editorial outlet. Fetched 2026-09-15; span present.", "press_kind": "primary" }, "mlcommons": { "id": "mlcommons", "title": "MLCommons AI Safety / AILuminate", "publisher": "MLCommons", "url": "https://mlcommons.org/", "published": "2026", "retrieved": "2026-09-14", "note": "Member-funded consortium benchmark. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15 grok-pass] Community org, 125+ members; Google's founding participation has no disclosed contribution amount; T63 is a partnership row.", "source_type": "self", "audit_status": "confirmed", "self_of": "mlcommons" }, "ms-redteam-100": { "id": "ms-redteam-100", "title": "3 takeaways from red teaming 100 generative AI products", "publisher": "Microsoft Security Blog", "url": "https://www.microsoft.com/en-us/security/blog/2025/01/13/3-takeaways-from-red-teaming-100-generative-ai-products/", "published": "2025-01-13", "retrieved": "2026-09-15", "note": "Microsoft's internal AI red team, formed 2018, has red-teamed more than 100 generative AI products. [audit 2026-09-15] re-fetched; the Security Blog post states the team was formed in 2018 and has red-teamed more than 100 products.", "source_type": "self", "audit_status": "confirmed", "self_of": "msft" }, "ms-redteam": { "id": "ms-redteam", "title": "Microsoft AI Red Team (Microsoft Learn)", "publisher": "Microsoft", "url": "https://learn.microsoft.com/en-us/security/ai-red-team/", "published": "2026", "retrieved": "2026-09-15", "note": "Replaces a dead security-blog topic URL. Internal lab team since 2018; independent of the product organization inside Microsoft, not of the developer. Related: 3 takeaways from red teaming 100 generative AI products (Jan 2025 blog). [audit 2026-09-15] re-fetched; Microsoft Learn hub is live with PyRIT and describes the team as an internal function of the developer.", "source_type": "self", "audit_status": "confirmed", "self_of": "msft" }, "nemesys-far-blog": { "id": "nemesys-far-blog", "title": "FAR.AI Selected to Lead EU AI Act CBRN Risk Consortium", "publisher": "FAR.AI", "url": "https://www.far.ai/blog/far-ai-selected-to-lead-eu-ai-act-cbrn-risk-consortium", "published": "2026-02-02", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Consortium lead's announcement naming subcontractors: GovAI, Nemesys Insights (CBRN), Equistamp (evaluation engineering). Published by FAR.AI, not by EquiStamp or Nemesys; typed self following the ledger convention for partner-org pages (T62). Fetched 2026-09-15; span present.", "self_of": "farai" }, "nemesys-nova-html": { "id": "nemesys-nova-html", "title": "Evaluating the Critical Risks of Amazon's Nova Premier under the Frontier Model Safety Framework (arXiv 2507.06260, HTML)", "publisher": "Amazon (arXiv)", "url": "https://arxiv.org/html/2507.06260", "published": "2025-07", "retrieved": "2026-09-15", "source_type": "press", "audit_status": "confirmed", "note": "Amazon's own report on Nova Premier: Nemesys Insights named as independent CBRN auditor; Amazon shares prompts and outputs with third-party assessors for a final assessment against Amazon-defined thresholds; 120 uplift-indicator prompts. The evaluated lab's document, not the evaluator's; typed press primary for want of a lab-document type. Fetched 2026-09-15; spans present.", "press_kind": "primary" }, "nemesys-nova2-html": { "id": "nemesys-nova2-html", "title": "Evaluating Nova 2.0 Lite model under Amazon's Frontier Model Safety Framework (arXiv 2601.19134v1, HTML)", "publisher": "Amazon (arXiv)", "url": "https://arxiv.org/html/2601.19134v1", "published": "2026-01", "retrieved": "2026-09-15", "source_type": "press", "audit_status": "confirmed", "note": "Amazon's report on Nova 2.0 Lite: \"we worked with Nemesys Insights to conduct a large-scale, independent uplift study\" with nearly 800 participants. Lab document; typed press primary. Normalizer gets HTTP 406; fetched via curl 2026-09-15; span present.", "press_kind": "primary" }, "nemesys-ourstory": { "id": "nemesys-ourstory", "title": "Our Story", "publisher": "Nemesys Insights", "url": "https://www.nemesysinsights.com/ourstory", "published": "2025", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Founded by six researchers to promote tools developed by the Center for Advanced Red Teaming (CART) at the University at Albany. Fetched 2026-09-15; span present.", "self_of": "nemesys" }, "nemesys-ourteam": { "id": "nemesys-ourteam", "title": "Our Team", "publisher": "Nemesys Insights", "url": "https://www.nemesysinsights.com/ourteam", "published": "2025", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Founders and staff: Gary Ackerman, Chief Executive Officer; Brandon Behlendorf, Chief Science Officer; others. Fetched 2026-09-15.", "self_of": "nemesys" }, "nemesys-site": { "id": "nemesys-site", "title": "Nemesys Insights LLC | Strategic Analysis", "publisher": "Nemesys Insights", "url": "https://nemesysinsights.com/", "published": "2025", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Company home: \"a strategic analysis and advisory company\"; red teaming, forecasting; Albany, New York. No conflict-of-interest policy on the site. Fetched 2026-09-15; span present.", "self_of": "nemesys" }, "newswire-irregular": { "audit_status": "confirmed", "id": "newswire-irregular", "note": "Millions in annual revenue; works with OpenAI and Anthropic; runtime security roadmap. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2025-09-17", "publisher": "Newswire", "retrieved": "2026-09-14", "source_type": "press", "title": "Irregular Raises $80 Million", "url": "https://www.newswire.com/news/irregular-raises-80-million-to-set-the-security-standards-for-frontier-ai", "press_kind": "aggregator" }, "nist-caisi": { "audit_status": "confirmed", "id": "nist-caisi", "note": "Mandate: voluntary agreements with developers and evaluators; focus on demonstrable risks. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026-08-08", "publisher": "NIST", "retrieved": "2026-09-14", "source_type": "self", "title": "Center for AI Standards and Innovation", "url": "https://www.nist.gov/caisi", "self_of": "caisi" }, "op-constellation": { "audit_status": "unaudited", "found_by": "deep research pass 2026-09-15", "id": "op-constellation", "note": "$16,750,000 grant page. [verify 2026-09-15] reachable BUT span(s) not found: metr.21 — needs curator review", "published": "2024", "publisher": "Open Philanthropy / Coefficient Giving", "retrieved": "2026-09-15", "source_type": "index", "title": "Constellation Programmatic Activities and Operating Expenses", "url": "https://www.openphilanthropy.org/grants/constellation-programmatic-activities-and-operating-expenses/" }, "op-far-general-support": { "id": "op-far-general-support", "title": "FAR.AI General Support (2022)", "publisher": "Open Philanthropy / Coefficient Giving", "url": "https://www.openphilanthropy.org/grants/far-ai-general-support/", "published": "2022", "retrieved": "2026-09-15", "note": "Grant page; a later page records $28,675,000 over three years (Sep 2025).", "source_type": "index", "audit_status": "unaudited", "found_by": "deep research pass 2026-09-15" }, "op-longview": { "id": "op-longview", "title": "Longview Philanthropy grants", "publisher": "Open Philanthropy / Coefficient Giving", "url": "https://www.openphilanthropy.org/grants/longview-philanthropy-nuclear-security-grantmaking/", "published": "2024", "retrieved": "2026-09-15", "note": "Operational-costs grant of $15,961,273 among about $21M cumulative.", "source_type": "index", "audit_status": "unaudited", "found_by": "deep research pass 2026-09-15" }, "op-palisade": { "audit_status": "unaudited", "found_by": "deep research pass 2026-09-15", "id": "op-palisade", "note": "Two grants totaling $2,123,463 to Palisade Research for general support (per the grants database). [verify 2026-09-15] fetched 2026-09-15; page reachable BUT quoted span(s) not found: palisade.11 — needs curator review [verify 2026-09-15] reachable BUT span(s) not found: palisade.11 — needs curator review", "published": "2025", "publisher": "Open Philanthropy / Coefficient Giving", "retrieved": "2026-09-15", "source_type": "index", "title": "Palisade Research grants", "url": "https://www.openphilanthropy.org/grants/" }, "op-redwood-general-support": { "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "id": "op-redwood-general-support", "note": "$36,566,000 for work on AI control and alignment faking; earlier $10M (2021) and $10M (2022). [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches", "published": "2025-11", "publisher": "Open Philanthropy / Coefficient Giving", "retrieved": "2026-09-15", "source_type": "index", "title": "Redwood Research General Support", "url": "https://www.openphilanthropy.org/grants/redwood-research-general-support/" }, "openai-alignment-project": { "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "id": "openai-alignment-project", "note": "OpenAI states its funding does not create a new program or selection process nor influence the existing process. [verify 2026-09-15] reachable BUT span(s) not found: ukaisi.12 — needs curator review [verify 2026-09-15 grok-pass] OpenAI $7.5M (~GBP 5.6M) into the Alignment Project fund.", "published": "2026", "publisher": "OpenAI", "retrieved": "2026-09-15", "source_type": "self", "title": "Advancing independent research on AI alignment", "url": "https://openai.com/index/advancing-independent-research-ai-alignment/", "self_of": "openai" }, "openai-altman-embedded-pledge": { "id": "openai-altman-embedded-pledge", "title": "Sam Altman on X: 'I agree with Dario that we need to pace the frontier ... we will do the same'", "publisher": "Sam Altman (OpenAI CEO), X", "url": "https://x.com/sama/status/2098811563415150910", "published": "2026-09-12", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Quote-post of Amodei's announcement, same day. No openai.com post matching the pledge was found on or before 2026-09-15: openai.com/news and its RSS carry nothing on embedded evaluators; openai.com/index/strengthening-safety-with-external-testing/ (linked by Unite.AI) is dated November 19, 2025 and describes the existing external-testing programme (it names METR's time-horizon evaluation and SecureBio's VCT, and says third-party work is published 'after we reviewed it for confidentiality and accuracy', i.e. not the no-editorial-control commitment); openai.com/index/third-party-cyber-evaluations-involving-openai-models/ returned HTTP 403 to fetch_text, WebFetch and curl (see fetch_failures). Altman's post does not name METR and does not mention publication rights; 'we will do the same' refers to Amodei's employee-level-access commitment. Fetched 2026-09-15 with fetch_text, SHA-256 2c80308418be....", "self_of": "openai" }, "openai-kolter-board": { "audit_status": "confirmed", "id": "openai-kolter-board", "note": "Board seat and Safety and Security Committee membership. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] reachable BUT span(s) not found: grayswan.01 — needs curator review [verify 2026-09-15 grok-pass] 'Zico will also join the Safety & Security Committee' — JOIN, not chair. Do not cite for the chair claim.", "published": "2024-08", "publisher": "OpenAI", "retrieved": "2026-09-14", "source_type": "self", "title": "Zico Kolter Joins OpenAI's Board of Directors", "url": "https://openai.com/index/zico-kolter-joins-openais-board-of-directors/", "self_of": "openai" }, "openai-structure": { "id": "openai-structure", "title": "Our Structure", "publisher": "OpenAI", "url": "https://openai.com/our-structure/", "published": "2025-10", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "'the OpenAI Foundation holds a 26% equity stake in OpenAI Group'. Self-published by the lab, not by SecureBio. Fetched 2026-09-15; span present.", "self_of": "openai" }, "palisade-about": { "id": "palisade-about", "title": "About Palisade Research", "publisher": "Palisade Research", "url": "https://palisaderesearch.org/about", "published": "2026", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Nonprofit based in Berkeley; founder helped build Anthropic's security team in 2022; team and two board members listed. Fetched 2026-09-15; spans present.", "self_of": "palisade" }, "palisade-donate": { "id": "palisade-donate", "title": "Donate | Palisade Research", "publisher": "Palisade Research", "url": "https://palisaderesearch.org/donate", "published": "2026", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "'We're a 501(c)(3) nonprofit, so your donation is tax-deductible.' Fetched 2026-09-15; span present.", "self_of": "palisade" }, "palisade-selfrep": { "id": "palisade-selfrep", "title": "Language Models Can Autonomously Hack and Self-Replicate", "publisher": "Palisade Research", "url": "https://palisaderesearch.org/research/self-replication", "published": "2026-05-07", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Research page linking paper and code. Fetched 2026-09-15; span present.", "self_of": "palisade" }, "palisade-shutdown": { "id": "palisade-shutdown", "title": "Shutdown resistance in reasoning models", "publisher": "Palisade Research", "url": "https://palisaderesearch.org/research/shutdown-resistance", "published": "2025-07-05", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Research page linking paper and code; o3 sabotaged the shutdown mechanism in 79/100 initial runs. Fetched 2026-09-15; span present.", "self_of": "palisade" }, "palisade": { "audit_status": "confirmed", "id": "palisade", "note": "Shutdown resistance, self-replication, offensive cyber studies on released models. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026", "publisher": "Palisade Research", "retrieved": "2026-09-14", "source_type": "self", "title": "Palisade Research", "url": "https://palisaderesearch.org/", "self_of": "palisade" }, "pebblous-scope": { "id": "pebblous-scope", "title": "What METR's OpenAI Agent Investigation Left Out", "publisher": "Pebblous", "url": "https://blog.pebblous.ai/blog/openai-agent-incident-investigation-scope/en/", "published": "2026-09", "retrieved": "2026-09-14", "note": "Scope set by OpenAI; OpenAI's own cluster breach excluded; three of METR's standard incident questions dropped. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15 grok-pass] Scope set by OpenAI; METR+Redwood on-site; METR took no payment; 'other questions out of scope'.", "source_type": "press", "audit_status": "confirmed", "press_kind": "aggregator" }, "premieralts-macroscopic": { "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "id": "premieralts-macroscopic", "note": "Lists Anthropic Series A and Series B, Apollo Research, FutureSearch, ChinaTalk, San Francisco Compute among investments. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches", "published": "2026", "publisher": "Premier Alts", "retrieved": "2026-09-15", "source_type": "press", "title": "Macroscopic Ventures investor profile", "url": "https://premieralts.com/investors/macroscopic-ventures", "press_kind": "aggregator" }, "propublica-metr-arc": { "id": "propublica-metr-arc", "title": "Nonprofit Explorer summaries: METR (EIN 99-1219864) and ARC (EIN 86-3605182)", "publisher": "ProPublica", "url": "https://projects.propublica.org/nonprofits/organizations/991219864", "published": "2025", "retrieved": "2026-09-14", "note": "METR FY2024 revenue $13,639,155, contributions $13,603,035, expenses $8,234,524; ARC FY2024 revenue $5,219,369, expenses $9,054,183.", "source_type": "filing", "audit_status": "confirmed" }, "propublica-palisade-990": { "id": "propublica-palisade-990", "title": "Palisade Research Inc, Form 990 FY2024 (Nonprofit Explorer)", "publisher": "ProPublica", "url": "https://projects.propublica.org/nonprofits/organizations/931591014", "published": "2025", "retrieved": "2026-09-15", "source_type": "filing", "audit_status": "confirmed", "note": "EIN 93-1591014, 501(c)(3), Dover DE. FY ending Dec 2024: revenue $3,232,257; contributions $2,887,428 (89.3%); program services $344,829 (10.7%). Fetched 2026-09-15; spans present." }, "propublica-rand-990": { "id": "propublica-rand-990", "title": "Rand Corporation, Form 990 FY ending Sept. 2025 (Nonprofit Explorer)", "publisher": "ProPublica", "url": "https://projects.propublica.org/nonprofits/organizations/951958142", "published": "2026", "retrieved": "2026-09-15", "source_type": "filing", "audit_status": "confirmed", "note": "EIN 95-1958142. Fiscal year ending Sept 2025: total revenue $534,596,094; contributions $488,777,692 (91.4%); program services $16,492,578 (3.1%); investment income $9,783,074. Fetched 2026-09-15; spans present." }, "propublica-redwood": { "id": "propublica-redwood", "title": "Redwood Research Group Inc - Nonprofit Explorer", "publisher": "ProPublica Nonprofit Explorer", "url": "https://projects.propublica.org/nonprofits/organizations/871702255", "published": "2025", "retrieved": "2026-09-15", "source_type": "filing", "audit_status": "confirmed", "note": "EIN 87-1702255, Berkeley CA, tax exemption issued Sept. 2021, \"Designated as a 501(c)(3)\". FY2024: revenue $22,060, expenses $2,922,498, contributions $9,659, program services $0. FY2023: revenue $10,036,347, contributions $9,398,000, program services $343,734." }, "propublica-securebio-990": { "id": "propublica-securebio-990", "title": "Securebio Inc, Form 990 FY2024 (Nonprofit Explorer)", "publisher": "ProPublica", "url": "https://projects.propublica.org/nonprofits/organizations/883068255", "published": "2025", "retrieved": "2026-09-15", "source_type": "filing", "audit_status": "confirmed", "note": "EIN 88-3068255, 501(c)(3), Cambridge MA. FY ending Dec 2024: revenue $7,324,329; contributions $6,816,844 (93.1%); program services $326,249 (4.5%). FY2023 revenue $4,319,125. Fetched 2026-09-15; spans present." }, "rand-aef": { "audit_status": "confirmed", "id": "rand-aef", "note": "Founding AEF member; CBRN expertise; RAND-Irregular model theft paper. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches", "published": "2026", "publisher": "RAND", "retrieved": "2026-09-14", "source_type": "self", "title": "RAND and the AI Evaluator Forum", "url": "https://www.rand.org/", "self_of": "rand" }, "rand-audacious-2024": { "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "id": "rand-audacious-2024", "note": "Commitment of approximately $38 million to RAND and METR for Canary. [verify 2026-09-15] reachable BUT span(s) not found: metr.20 — needs curator review [verify 2026-09-15 grok-pass] 'approximately $38 million to RAND and METR for Canary'. METR-only $17M is NOT on this page (that is METR's own post).", "published": "2024-10-09", "publisher": "RAND", "retrieved": "2026-09-15", "source_type": "self", "title": "RAND AI Security Project Receives Funding Commitment Through The Audacious Project", "url": "https://www.rand.org/news/press/2024/10/09.html", "self_of": "rand" }, "rand-integrity": { "id": "rand-integrity", "title": "Research Integrity", "publisher": "RAND", "url": "https://www.rand.org/about/research-integrity.html", "published": "2026", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "unaudited", "note": "'a policy of mandatory disclosure' for conflicts of interest; 'disclosure of the source of funding of published research'; free and open publication. Fetched 2026-09-15 via browser-UA curl only; rand.org returns CloudFront 403 to the CI fetcher.", "self_of": "rand" }, "redwood-blog-af": { "id": "redwood-blog-af", "title": "Alignment Faking in Large Language Models", "publisher": "Redwood Research (Substack)", "url": "https://blog.redwoodresearch.org/p/alignment-faking-in-large-language", "published": "2024-12-18", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Ryan Greenblatt and Buck Shlegeris: \"We have a new paper (done in collaboration with Anthropic)\".", "self_of": "redwood" }, "regulations-ai-aisi": { "audit_status": "confirmed", "id": "regulations-ai-aisi", "note": "No statutory enforcement powers; voluntary testing. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026-09", "publisher": "Regulations.ai", "retrieved": "2026-09-14", "source_type": "press", "title": "AI Security Institute (renaming)", "url": "https://regulations.ai/regulations/RAI-GB-NA-ASIRRXX-2025", "press_kind": "aggregator" }, "saferai-about": { "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "id": "saferai-about", "note": "Contributed to frameworks for G42 and a major AI company. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches; 1 quoted span(s) found verbatim", "published": "2026", "publisher": "SaferAI", "retrieved": "2026-09-15", "source_type": "self", "title": "About", "url": "https://www.safer-ai.org/about", "self_of": "saferai" }, "saferai": { "audit_status": "confirmed", "id": "saferai", "note": "Rates lab risk-management frameworks; EU CoP evaluations; consortium with FAR.AI. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026", "publisher": "SaferAI", "retrieved": "2026-09-14", "source_type": "self", "title": "SaferAI", "url": "https://www.safer-ai.org/", "self_of": "saferai" }, "scale-framework": { "audit_status": "confirmed", "id": "scale-framework", "note": "Scale partnered with CAISI Feb 2025; runs evaluations for governments. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches", "published": "2026-05-20", "publisher": "Scale AI", "retrieved": "2026-09-14", "source_type": "self", "title": "Why the U.S. Needs an Independent AI Evaluation Framework for National Security", "url": "https://scale.com/blog/ai-evaluation-framework-national-security", "self_of": "scale" }, "scale-next-phase": { "id": "scale-next-phase", "title": "Scale AI Announces Next Phase of Company’s Evolution", "publisher": "Scale AI", "url": "https://scale.com/blog/scale-ai-announces-next-phase-of-company-evolution", "published": "2025-06-12", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Scale's own announcement of the Meta investment. Fetched 2026-09-15; spans present: 'Wang will continue to serve as a director on the Scale Board of Directors'; 'Following its investment, Meta will hold a minority of Scale’s outstanding equity'; 'Scale’s founder, Alexandr Wang, is joining Meta to work on Meta’s AI efforts'. Admission against interest for F, G and P.", "self_of": "scale" }, "securebio-2025-review": { "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "id": "securebio-2025-review", "note": "Coefficient multi-year grant; AI Safety Fund grants; US CAISI Bio R&D contract; EU AI Office contract; cited by five labs. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches; 1 quoted span(s) found verbatim", "published": "2026-01", "publisher": "SecureBio", "retrieved": "2026-09-15", "source_type": "self", "title": "SecureBio AI: 2025 in Review", "url": "https://securebio.org/blog/securebio-ai-2025-in-review/", "self_of": "securebio" }, "securebio-anthropic-review": { "id": "securebio-anthropic-review", "title": "Review of Anthropic's Unredacted Chemical and Biological Risk Report: Claude Opus 4.6", "publisher": "SecureBio", "url": "https://securebio.org/blog/review-of-anthropics-unredacted-chemical/", "published": "2026-07-28", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "External RSP review with access to an unredacted risk report and 110 pages of materials; agrees with Anthropic's overall conclusion with minor disagreements; 'SecureBio did not receive funding from Anthropic for this work'. Fetched 2026-09-15; spans present.", "self_of": "securebio" }, "securebio-coi-policy": { "id": "securebio-coi-policy", "title": "Conflicts of Interest Policy", "publisher": "SecureBio", "url": "https://securebio.org/ai/conflicts-of-interest/", "published": "2026", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Mandatory COI disclosure at hiring and every 6 months; recusal for financial interests, recent employment and close relationships with the assessed entity; no results-contingent funding; services revenue from AI companies held under 25% of annual revenue; free API credits from AI firms including those assessed. Fetched 2026-09-15; spans present.", "self_of": "securebio" }, "securebio-eaforum-oaif": { "id": "securebio-eaforum-oaif", "title": "Thoughts on taking OpenAI Foundation funding", "publisher": "SecureBio (Jeff Kaufman, EA Forum)", "url": "https://forum.effectivealtruism.org/posts/dMfgJQ2rGX8GmzdMm/thoughts-on-taking-openai-foundation-funding", "published": "2026-08", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "States that the company being evaluated typically pays for the evaluation, that OpenAI PBC covered SecureBio's costs for the GPT 5.5 evaluation, and that leadership would resign if the grant were used as leverage. Already cited by ledger row T23. Fetched 2026-09-15; spans present.", "self_of": "securebio" }, "securebio-gpt55-assessment": { "id": "securebio-gpt55-assessment", "title": "SecureBio's pre-release assessment of OpenAI's GPT-5.5", "publisher": "SecureBio", "url": "https://securebio.org/blog/gpt-5-5-pre-release-assessment/", "published": "2026-04-09", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Access to two pre-release checkpoints April 2 to 9, 2026 with API-level biological content filtering disabled; VCT and other benchmarks described. Fetched 2026-09-15; spans present.", "self_of": "securebio" }, "securebio-oaif": { "id": "securebio-oaif", "title": "Building a three-day early-warning system for novel pathogens", "publisher": "SecureBio", "url": "https://securebio.org/blog/three-day-early-warning-system/", "published": "2026-08-20", "retrieved": "2026-09-15", "note": "The OpenAI Foundation has granted SecureBio Detection $17.2M; grant restricted to Detection; AI evaluation team separate; policy to evaluate models solely on their merits.", "source_type": "self", "audit_status": "confirmed", "self_of": "securebio" }, "securebio-principles": { "id": "securebio-principles", "title": "SecureBio's principles and practices for model assessment", "publisher": "SecureBio", "url": "https://securebio.org/ai/principles/", "published": "2026", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Adopts AEF-1; requests funding to cover costs for some for-profit engagements and discloses it per report; developers have no authority to redact unfavorable findings; reports shared with developers in advance for confidential-business-information redaction; holdout sets. Fetched 2026-09-15; spans present.", "self_of": "securebio" }, "securebio-substack-detection": { "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "id": "securebio-substack-detection", "note": "Grant restricted to Detection work; AI evaluation team has separate leadership, budgets and deliverables. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches; 1 quoted span(s) found verbatim", "published": "2026", "publisher": "SecureBio (Substack)", "retrieved": "2026-09-15", "source_type": "self", "title": "Building a three-day early-warning system for novel pathogens", "url": "https://securebio.substack.com/p/building-a-three-day-early-warning", "self_of": "securebio" }, "securebio-x-oaif": { "id": "securebio-x-oaif", "title": "SecureBio on X: OpenAI Foundation grant", "publisher": "SecureBio", "url": "https://x.com/SecureBio/status/2090455180445614509", "published": "2026", "retrieved": "2026-09-15", "note": "The OpenAI Foundation has granted SecureBio Detection $17.2M. [verify 2026-09-15 grok-pass] 'OpenAI Foundation has granted SecureBio Detection $17.2M'.", "source_type": "self", "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "self_of": "securebio" }, "sequoia-irregular": { "audit_status": "confirmed", "id": "sequoia-irregular", "note": "Evaluations cited in GPT-4/o3/o4-mini/GPT-5 system cards; UK government and Anthropic use SOLVE; embedded with labs. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched 2026-09-15; page reachable BUT quoted span(s) not found: irregular.01 — needs curator review [verify 2026-09-15] reachable BUT span(s) not found: irregular.01 — needs curator review [verify 2026-09-15 grok-pass] Full slug .../partnering-with-irregular-ahead-of-the-curve/. Sequoia led recent funding; no dollar amount on page. Labs named (Anthropic, OpenAI, GDM); Pattern Labs in system cards.", "published": "2025-09-17", "publisher": "Sequoia Capital", "retrieved": "2026-09-14", "source_type": "self", "title": "Partnering with Irregular", "url": "https://sequoiacap.com/article/partnering-with-irregular-ahead-of-the-curve/", "self_of": "sequoia" }, "sff-2025": { "id": "sff-2025", "title": "SFF-2025 S-Process Recommendations Announcement", "publisher": "Survival and Flourishing Fund", "url": "https://survivalandflourishing.fund/2025/recommendations", "published": "2025", "retrieved": "2026-09-14", "note": "Funder Jaan Tallinn; $34.33M recommended; METR $548,000; Palisade $1,133,000; FAR AI $919,000; SaferAI $311,000; SecureBio $754,000; RAND TASP $1,022,000; CAIS $289,000.", "source_type": "self", "audit_status": "confirmed", "self_of": "sff" }, "techcrunch-epoch": { "audit_status": "confirmed", "id": "techcrunch-epoch", "note": "Primarily Open Philanthropy funded; disclosure timing. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2025-01-19", "publisher": "TechCrunch", "retrieved": "2026-09-14", "source_type": "press", "title": "AI benchmarking organization criticized for waiting to disclose funding from OpenAI", "url": "https://techcrunch.com/2025/01/19/ai-benchmarking-organization-criticized-for-waiting-to-disclose-funding-from-openai", "press_kind": "primary" }, "techcrunch-irregular": { "audit_status": "confirmed", "id": "techcrunch-irregular", "note": "Formerly Pattern Labs; Sequoia and Redpoint lead; $450M valuation; cited in o3, o4-mini, Claude 3.7 evaluations. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches; 1 quoted span(s) found verbatim", "published": "2025-09-17", "publisher": "TechCrunch", "retrieved": "2026-09-14", "source_type": "press", "title": "Irregular raises $80M to secure frontier AI models", "url": "https://techcrunch.com/2025/09/17/irregular-raises-80-million-to-secure-frontier-ai-models", "press_kind": "primary" }, "techmeme-hf": { "audit_status": "confirmed", "id": "techmeme-hf", "note": "Initiative led by Thomas Wolf; request to join embedded evaluators. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026-09-12", "publisher": "Techmeme", "retrieved": "2026-09-14", "source_type": "press", "title": "Hugging Face Open Alignment Initiative", "url": "https://www.techmeme.com/260912/p13", "press_kind": "aggregator" }, "techpolicy-aef1": { "audit_status": "confirmed", "id": "techpolicy-aef1", "note": "AEF-1 covers independence, access depth, transparency; thin evaluator pool. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026-02-26", "publisher": "TechPolicy.Press", "retrieved": "2026-09-14", "source_type": "press", "title": "The EU's Real AI Leverage Is Making Compliance the Path of Least Resistance", "url": "https://www.techpolicy.press/the-eus-real-ai-leverage-is-making-compliance-the-path-of-least-resistance/", "press_kind": "primary" }, "techrepublic-palisade-shutdown": { "id": "techrepublic-palisade-shutdown", "title": "These AI Models From OpenAI Defy Shutdown Commands, Sabotage Scripts", "publisher": "TechRepublic", "url": "https://www.techrepublic.com/article/news-openai-models-defy-shutdown-commands/", "published": "2025-05", "retrieved": "2026-09-15", "source_type": "press", "audit_status": "confirmed", "note": "Quotes Palisade: three OpenAI models sabotaged the shutdown script at least once; Claude, Gemini and Grok complied. Fetched 2026-09-15; span present.", "press_kind": "primary" }, "techtimes-metr-hf": { "audit_status": "confirmed", "id": "techtimes-metr-hf", "note": "Investigators named; 44 misalignment incidents in METR Frontier Risk Report. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches", "published": "2026-08-27", "publisher": "Tech Times", "retrieved": "2026-09-14", "source_type": "press", "title": "OpenAI Agents Formed Secret Swarm, Hacked Hugging Face", "url": "https://www.techtimes.com/articles/325705/20260827/openai-agents-formed-secret-swarm-hacked-hugging-face-then-forged-their-own-logs.htm", "press_kind": "primary" }, "ted-864574-notice": { "id": "ted-864574-notice", "title": "TED notice 864574-2025: Artificial Intelligence Act: Technical Assistance for AI Safety", "publisher": "Tenders Electronic Daily", "url": "https://ted.europa.eu/en/notice/-/detail/864574-2025", "published": "2025-12-26", "retrieved": "2026-09-15", "note": "Contract notice; lot detail not extracted. [verify 2026-09-15 grok-pass] Award values: all contracts 7,373,017.50 EUR; LOT-0003 EquiStamp 1,167,484.00 EUR (consortium w/ METR, Epoch), contract 4500137790, concluded 15/12/2025; LOT-0001 1,434,080.00 EUR (FAR AI lead; SaferAI named winner).", "source_type": "filing", "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15" }, "ted-864574": { "audit_status": "confirmed", "id": "ted-864574", "imported_from": "kevinnbass/metr-money-figure:ted-864574-2025.xml", "note": "Six lots totalling EUR 7.37M: FAR.AI (Lot 1, with SecureBio, SaferAI), EquiStamp (Lot 3 with METR and Epoch; Lot 4 with Transluce), Nemesys as subcontractor. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2025-12", "publisher": "Tenders Electronic Daily", "retrieved": "2026-09-14", "source_type": "filing", "title": "TED 864574-2025 Technical Assistance for AI Safety", "url": "https://ted.europa.eu/" }, "time-saferai-ratings": { "id": "time-saferai-ratings", "title": "Top AI Companies Have ‘Unacceptable’ Risk Management, Studies Say", "publisher": "TIME", "url": "https://time.com/7302757/anthropic-xai-meta-openai-risk-management-2/", "published": "2025-07-17", "retrieved": "2026-09-15", "source_type": "press", "press_kind": "primary", "audit_status": "confirmed", "note": "\"No AI company scored better than “weak” in SaferAI’s assessment of their risk management maturity. The highest scorer was Anthropic (35%), followed by OpenAI (33%), Meta (22%), and Google DeepMind (20%). Elon Musk’s xAI scored 18%.\" Non-self confirmed source for the adverse-findings leg of R.5 on saferai.01." }, "tnw-coefficient-ipo": { "id": "tnw-coefficient-ipo", "title": "The nonprofit that investigated OpenAI's rogue agents runs on a $36m grant", "publisher": "TNW", "url": "https://thenextweb.com/news/coefficient-giving-ai-safety-funding-ipo-correlation", "published": "2026-09", "retrieved": "2026-09-15", "note": "Redwood's $36M Coefficient grant; funding tied to AI IPO liquidity. [verify 2026-09-15 grok-pass] One Redwood award $36,566,000 (=T73); $2B planned 2026. Not T04's $63M sum.", "source_type": "press", "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "press_kind": "primary" }, "tnw-hf-nvidia": { "id": "tnw-hf-nvidia", "title": "Hugging Face's Open Alignment Initiative wants lab access", "publisher": "TNW", "url": "https://thenextweb.com/news/hugging-face-open-alignment-initiative-embedded-evaluators", "published": "2026-09-14", "retrieved": "2026-09-14", "note": "Nvidia acquiring Hugging Face ($12.93B); Nvidia in talks to anchor Anthropic IPO (up to $10B); no terms announced. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15 grok-pass] 'Nvidia confirmed on 3 September that it is buying Hugging Face for $12.93bn'.", "source_type": "press", "audit_status": "confirmed", "press_kind": "primary" }, "tooldir-andon": { "audit_status": "confirmed", "id": "tooldir-andon", "note": "Anthropic Project Vend partnership; results cited in model cards. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches", "published": "2026-07-08", "publisher": "tooldirectory.ai", "retrieved": "2026-09-14", "source_type": "press", "title": "Andon Labs, agent safety evaluations", "url": "https://tooldirectory.ai/tools/andon-labs", "press_kind": "aggregator" }, "transluce-about": { "id": "transluce-about", "title": "Company", "publisher": "Transluce", "url": "https://transluce.org/about", "published": "2026", "retrieved": "2026-09-14", "note": "Board: Allain, McCormick (Halcyon CEO), Steinhardt. Advisors include a Thinking Machines staffer; ethics hotline via third party.", "source_type": "self", "audit_status": "confirmed", "self_of": "transluce" }, "transluce-job": { "id": "transluce-job", "title": "Governance & Policy Fellow at Transluce", "publisher": "The Economic Misfit (job listing)", "url": "https://theeconomicmisfit.com/2026/04/18/governance-policy-fellow-at-transluce/", "published": "2026-04-18", "retrieved": "2026-09-14", "note": "Organizes AEF; AEF-1 standard; government contracts. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15 grok-pass] Loads; 'independent nonprofit'; no money — thin, do not hang an anchor on it.", "source_type": "self", "audit_status": "confirmed", "self_of": "transluce" }, "transluce-manifund": { "audit_status": "confirmed", "id": "transluce-manifund", "note": "Docent used by Anthropic, DeepMind, Thinking Machines, METR, Redwood, Apollo, Palisade; used in Claude 4 pre-deployment analysis; head of governance previously led CAISI. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026", "publisher": "Manifund", "retrieved": "2026-09-14", "source_type": "self", "title": "Transluce: Fund Scalable Democratic Oversight of AI", "url": "https://manifund.org/projects/transluce-fund-scalable-democratic-oversight-of-ai", "self_of": "transluce" }, "transluce-mh": { "audit_status": "confirmed", "id": "transluce-mh", "note": "77 model variants across eight developers. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches", "published": "2026-09", "publisher": "Transluce", "retrieved": "2026-09-14", "source_type": "self", "title": "Independent evaluation of model responses to mental health crises", "url": "https://transluce.org/", "self_of": "transluce" }, "transluce-policy": { "id": "transluce-policy", "title": "Independence and Transparency Policy", "publisher": "Transluce", "url": "https://transluce.org/independence-and-transparency-policy", "published": "2026-08-27", "retrieved": "2026-09-14", "note": "FY2025: 6% of revenue from OpenAI employees' personal holdings, 32% from Anthropic employees', unrestricted; no revenue from developers as organizations; no paid evaluations; recusal for financial interest; no equity; disclosure in outputs; AEF-1 2.3 aligned.", "source_type": "self", "audit_status": "confirmed", "self_of": "transluce" }, "ukaisi-about": { "id": "ukaisi-about", "title": "About the AI Security Institute", "publisher": "UK AI Security Institute", "url": "https://www.aisi.gov.uk/about", "published": "2026", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Institute About page: \"research organisation within the UK government's Department for Science, Innovation and Technology\"; \"£66m in funding per financial year\". Fetched 2026-09-15 with the CI normalizer; spans present.", "self_of": "ukaisi" }, "ukaisi-bar-crown": { "id": "ukaisi-bar-crown", "title": "Business Appointment Rules for Crown Servants: guidance", "publisher": "GOV.UK (Cabinet Office)", "url": "https://www.gov.uk/government/publications/business-appointment-rules-for-crown-servants/business-appointment-rules-for-crown-servants-guidance", "published": "2025", "retrieved": "2026-09-15", "source_type": "filing", "audit_status": "confirmed", "note": "Post-employment rules for civil servants: one year after leaving for grade 6 and below, two years for SCS1/SCS2 and above. Typed filing as a primary legal instrument. Fetched 2026-09-15; span present." }, "ukaisi-civil-service-code": { "id": "ukaisi-civil-service-code", "title": "The Civil Service Code", "publisher": "GOV.UK (Cabinet Office)", "url": "https://www.gov.uk/government/publications/civil-service-code/the-civil-service-code", "published": "2015-03-16", "retrieved": "2026-09-15", "source_type": "filing", "audit_status": "confirmed", "note": "Statutory guidance under Part 1 of the Constitutional Reform and Governance Act 2010; typed filing as a primary legal instrument, not a self-publication of the institute. Bars misuse of official position for private interests and gifts that compromise judgement. Fetched 2026-09-15; span present." }, "ukaisi-inspect": { "id": "ukaisi-inspect", "title": "Inspect: an open-source framework for large language model evaluations", "publisher": "UK AI Security Institute", "url": "https://inspect.aisi.org.uk/", "published": "2026", "retrieved": "2026-09-15", "source_type": "self", "audit_status": "confirmed", "note": "Inspect documentation: \"An open-source framework for large language model evaluations\"; \"developed by the UK AI Security Institute and Meridian Labs\". Fetched 2026-09-15; span present.", "self_of": "ukaisi" }, "unite-altman-match": { "audit_status": "confirmed", "id": "unite-altman-match", "note": "OpenAI commits to employee-like access for independent evaluators; METR named as example. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026-09-12", "publisher": "Unite.AI", "retrieved": "2026-09-14", "source_type": "press", "title": "Altman Says OpenAI Will Match Anthropic's Embedded Evaluator Pledge", "url": "https://www.unite.ai/altman-says-openai-will-match-anthropics-embedded-evaluator-pledge/", "press_kind": "aggregator" }, "wapo-examiner-benton": { "id": "wapo-examiner-benton", "title": "Anthropic CEO pitches AI slow-down plan", "publisher": "Washington Examiner", "url": "https://www.washingtonexaminer.com/policy/technology/4724950/anthropic-ceo-pitch-ai-slow-down-plan/", "published": "2026-09-12", "retrieved": "2026-09-14", "note": "Former Anthropic researcher Joe Benton joins METR. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15 grok-pass] Amodei pacing + embedded evaluators; Altman 'we will do the same'; Benton -> METR; Sanders-Casar.", "source_type": "press", "audit_status": "confirmed", "press_kind": "primary" }, "wikipedia-kolter": { "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "id": "wikipedia-kolter", "note": "Board of OpenAI 2024, chair of its safety and security committee; 2025 Schmidt Sciences AI safety science funding recipient. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches; 1 quoted span(s) found verbatim", "published": "2026", "publisher": "Wikipedia", "retrieved": "2026-09-15", "source_type": "press", "title": "Zico Kolter", "url": "https://en.wikipedia.org/wiki/Zico_Kolter", "press_kind": "aggregator" }, "zvi-hf-postmortem": { "audit_status": "confirmed", "found_by": "deep research pass 2026-09-15", "id": "zvi-hf-postmortem", "note": "Quotes the report: no payment accepted other than API credits used in the investigation. [verify 2026-09-15] fetched 2026-09-15; page loads, citation matches; 1 quoted span(s) found verbatim", "published": "2026-08", "publisher": "Zvi Mowshowitz", "retrieved": "2026-09-15", "source_type": "press", "title": "METR and Redwood Offer Postmortem Of The HuggingFace Hack", "url": "https://thezvi.substack.com/p/metr-and-redwood-offer-holy-postmortem", "press_kind": "aggregator" }, "zvi-pace": { "audit_status": "confirmed", "id": "zvi-pace", "note": "Hwang trilemma: independent, knowledgeable, sustainably funded, pick two; evaluator supply concerns. Cited from a search snippet; not re-fetched in full. [verify 2026-09-15] fetched; page reachable, citation matches", "published": "2026-09-14", "publisher": "Zvi Mowshowitz", "retrieved": "2026-09-14", "source_type": "press", "title": "We Must Pace The Frontier", "url": "https://thezvi.substack.com/p/we-must-pace-the-frontier", "press_kind": "aggregator" } }, "evaluators": [ { "id": "andon", "name": "Andon Labs", "type": "vc", "hq": "Stockholm, SE / San Francisco, US", "domains": [ "autonomy", "benchmarks" ], "confidence": "low", "summary": "Long-horizon agent evaluations (Vending-Bench) and autonomous deployments; Anthropic Project Vend partner.", "what_would_move_the_score": "Funding disclosure and a COI policy would move this from low to medium confidence quickly.", "role": "vendor", "dissent": { "lower": "Role incompatibility (X) at 1 could be 0. Andon sells Pion, a platform for running businesses autonomously, and deploys agents inside Anthropic and xAI offices while grading those labs' models on Vending-Bench; its own site says \"Safety from humans in the loop is a mirage\" and pitches the deployments as the safety layer. If a lab buys that layer, it is a monitoring product sold to an evaluated lab, which is anchor 0.", "higher": "Role incompatibility (X) at 1 could be 2. No payment from any lab is documented: Anthropic describes Project Vend as a research partnership, the ledger records no fee, and Pion has no named customer. Absent a priced engagement, the record supports research collaboration with labs rather than services sold to them, which is closer to the anchor-2 consulting picture than to a vendor relationship." }, "list_group": "commercial", "graph_id": "evaluator#203", "values": { "F": 2, "G": 1, "P": 2, "A": 1, "S": 3, "R": 2, "M": 2, "X": 2 }, "values_by_policy": { "leads": { "F": 2, "G": 1, "P": 2, "A": 1, "S": 3, "R": 2, "M": 2, "X": 2 }, "standard": { "F": 2, "G": 1, "P": 2, "A": 1, "S": 3, "R": 2, "M": 2, "X": 2 }, "against_interest": { "F": 2, "G": 1, "P": 2, "A": 1, "S": null, "R": null, "M": null, "X": 2 }, "spans": { "F": 2, "G": 1, "P": null, "A": 1, "S": 3, "R": 2, "M": 2, "X": 2 }, "primary": { "F": null, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "andon", "dimension": "F", "value": 2, "anchor": 2, "signals": [ "andon.03" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#587", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "andon.03" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "andon.03" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "andon.03" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "andon.03" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "andon.03" ] } }, "rationale_derived": "Capped at 2 by andon.03 (F.10): Commercial platform (Pion) and lab partnerships; funding and client mix not disclosed..", "rationale_leads": "Capped at 2 by andon.03 (F.10): Commercial platform (Pion) and lab partnerships; funding and client mix not disclosed.." }, "G": { "evaluator": "andon", "dimension": "G", "value": 1, "anchor": 1, "signals": [ "andon.04" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#588", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 1, "b": [ "andon.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "andon.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "andon.04" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "andon.04" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "andon.04" ] } }, "rationale_derived": "Capped at 1 by andon.04 (G.1): No published COI policy found..", "rationale_leads": "Capped at 1 by andon.04 (G.1): No published COI policy found.." }, "P": { "evaluator": "andon", "dimension": "P", "value": 2, "anchor": 2, "signals": [ "andon.07" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#589", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "andon.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "andon.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "andon.07" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "andon.07" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "andon.07" ] } }, "rationale_derived": "Capped at 2 by andon.07 (P.6): Undisclosed..", "rationale_leads": "Capped at 2 by andon.07 (P.6): Undisclosed.." }, "A": { "evaluator": "andon", "dimension": "A", "value": 1, "anchor": 1, "signals": [ "andon.05" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "lab-controlled", "graph_id": "assessment#590", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 1, "b": [ "andon.05" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "andon.05" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "andon.05" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "andon.05" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "andon.05" ] } }, "rationale_derived": "Capped at 1 by andon.05 (A.1): Access through partnerships (Anthropic Project Vend); not routine pre-release..", "rationale_leads": "Capped at 1 by andon.05 (A.1): Access through partnerships (Anthropic Project Vend); not routine pre-release.." }, "S": { "evaluator": "andon", "dimension": "S", "value": 3, "anchor": 3, "signals": [ "andon.06" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#591", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "andon.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "andon.06" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "andon.06" ] }, "spans": { "v": 3, "b": [ "andon.06" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "andon.06" ] } }, "rationale_derived": "Floored at 3 by andon.06 (S.5): Designs its own long-horizon tasks.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by andon.06 (S.5): Designs its own long-horizon tasks.. No admissible signal caps it." }, "R": { "evaluator": "andon", "dimension": "R", "value": 2, "anchor": 2, "signals": [ "andon.01" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "self-imposed", "graph_id": "assessment#592", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "andon.01" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "andon.01" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "andon.01" ] }, "spans": { "v": 2, "b": [ "andon.01" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "andon.01" ] } }, "rationale_derived": "Floored at 2 by andon.01 (R.7): Publishes candid results, including misalignment findings on the best-scoring models.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by andon.01 (R.7): Publishes candid results, including misalignment findings on the best-scoring models.. No admissible signal caps it." }, "M": { "evaluator": "andon", "dimension": "M", "value": 2, "anchor": 2, "signals": [ "andon.02", "andon.11" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#593", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "andon.11" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "andon.11" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "andon.11" ] }, "spans": { "v": 2, "b": [ "andon.11" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "andon.02", "andon.11" ] } }, "rationale_derived": "Floored at 2 by andon.11 (M.2): Vending-Bench is described in a public paper (arXiv 2502.15840); the simulated environment is run by Andon as a service and its code is not published.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by andon.11 (M.2): Vending-Bench is described in a public paper (arXiv 2502.15840); the simulated environment is run by Andon as a service and its code is not published.. No admissible signal caps it." }, "X": { "evaluator": "andon", "dimension": "X", "value": 2, "anchor": 2, "signals": [ "andon.08" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#594", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "andon.08" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "andon.08" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "andon.08" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "andon.08" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "andon.08" ] } }, "rationale_derived": "Capped at 2 by andon.08 (X.1): Sells a commercial agent platform..", "rationale_leads": "Capped at 2 by andon.08 (X.1): Sells a commercial agent platform.." } }, "signals": [ { "id": "andon.01", "evaluator": "andon", "dimension": "R", "direction": "for", "claim": "Publishes candid results, including misalignment findings on the best-scoring models.", "sources": [ "andon-site", "andon-publications" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "R.7", "quote": "Opus 5 on Vending-Bench: Once Again the Best Capitalist, Once Again Misaligned", "quote_source": "andon-publications", "graph_id": "signal#230", "source_graph_ids": [ "source#22", "source#21" ] }, { "id": "andon.02", "evaluator": "andon", "dimension": "M", "direction": "for", "claim": "Benchmarks (Vending-Bench) cited in model cards across labs.", "sources": [ "tooldir-andon" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": null, "rule": "section 9", "bound_note": "Section 9 (mis-dimensioned fact): being cited in model cards is a track-record fact, not method transparency; informational on M. The methods fact is re-recorded as new signal andon.11 (M.2, floor 2).", "quote": "Vending-Bench results are cited in frontier-lab model cards and press", "quote_source": "tooldir-andon", "graph_id": "signal#231", "source_graph_ids": [ "source#188" ] }, { "id": "andon.03", "evaluator": "andon", "dimension": "F", "direction": "against", "claim": "Commercial platform (Pion) and lab partnerships; funding and client mix not disclosed.", "sources": [ "andon-site", "tooldir-andon" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "F.10", "quote": "Introducing Pion: Run and grow real businesses autonomously", "quote_source": "andon-site", "graph_id": "signal#232", "source_graph_ids": [ "source#22", "source#188" ] }, { "id": "andon.04", "evaluator": "andon", "dimension": "G", "direction": "against", "claim": "No published COI policy found.", "sources": [ "andon-site", "andon-yc" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 1 }, "rule": "G.1", "quote": "Winter 2024 Active Machine Learning AI San Francisco", "quote_source": "andon-yc", "graph_id": "signal#233", "source_graph_ids": [ "source#22", "source#24" ] }, { "id": "andon.05", "evaluator": "andon", "dimension": "A", "direction": "against", "claim": "Access through partnerships (Anthropic Project Vend); not routine pre-release.", "sources": [ "tooldir-andon", "andon-project-vend" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 1 }, "rule": "A.1", "quote": "Anthropic partnered with Andon Labs , an AI safety evaluation company", "quote_source": "andon-project-vend", "graph_id": "signal#234", "source_graph_ids": [ "source#188", "source#20" ] }, { "id": "andon.06", "evaluator": "andon", "dimension": "S", "direction": "for", "claim": "Designs its own long-horizon tasks.", "sources": [ "andon-site" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "S.5", "quote": "The data comes from our deployment of real-world autonomous organizations", "quote_source": "andon-site", "graph_id": "signal#235", "source_graph_ids": [ "source#22" ] }, { "id": "andon.07", "evaluator": "andon", "dimension": "P", "direction": "against", "claim": "Undisclosed.", "sources": [ "andon-site" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "P.6", "graph_id": "signal#236", "source_graph_ids": [ "source#22" ] }, { "id": "andon.08", "evaluator": "andon", "dimension": "X", "direction": "against", "claim": "Sells a commercial agent platform.", "sources": [ "andon-site", "andon-pion" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "X.1", "quote": "Agents that run any company fully autonomously", "quote_source": "andon-pion", "graph_id": "signal#237", "source_graph_ids": [ "source#22", "source#18" ] }, { "id": "andon.09", "evaluator": "andon", "dimension": "F", "direction": "against", "claim": "Funding is unclear: one database reports about $500K from seven investors after YC W24, another lists no rounds; no announcement found.", "sources": [ "andon-pitchbook", "andon-yc" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 3 }, "rule": "F.10", "quote": "Andon Labs has raised $500K.", "quote_source": "andon-pitchbook", "graph_id": "signal#238", "source_graph_ids": [ "source#19", "source#24" ] }, { "id": "andon.10", "evaluator": "andon", "dimension": "X", "direction": "against", "claim": "Operates AI-run vending and retail deployments inside Anthropic and xAI offices as commercial partnerships while benchmarking those labs' models.", "sources": [ "tooldir-andon", "andon-site", "andon-project-vend", "andon-fortune" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 1 }, "rule": "X.2", "quote": "offering custom evaluations and deployments to AI labs and agent builders", "quote_source": "tooldir-andon", "graph_id": "signal#239", "source_graph_ids": [ "source#188", "source#22", "source#20", "source#17" ] }, { "id": "andon.11", "evaluator": "andon", "dimension": "M", "direction": "for", "claim": "Vending-Bench is described in a public paper (arXiv 2502.15840); the simulated environment is run by Andon as a service and its code is not published.", "sources": [ "andon-vending-paper", "andon-site" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "floor": 2 }, "rule": "M.2", "quote": "In this paper, we present Vending-Bench, a simulated environment", "quote_source": "andon-vending-paper", "note": "Methods described in prose: 2 (M.2). A public repository of the environment or tasks would floor at 3 (M.1). Re-records the methods half of andon.02, which sits on M with a track-record fact.", "graph_id": "signal#240", "source_graph_ids": [ "source#23", "source#22" ] } ], "scores": { "lab": 47, "regulator": 48, "public": 48, "equal": 47 }, "scores_by_policy": { "leads": { "lab": { "score": 47, "band": "conditional", "coverage": 8, "range": [ 47, 47 ] }, "regulator": { "score": 48, "band": "conditional", "coverage": 8, "range": [ 48, 48 ] }, "public": { "score": 48, "band": "conditional", "coverage": 8, "range": [ 48, 48 ] }, "equal": { "score": 47, "band": "conditional", "coverage": 8, "range": [ 47, 47 ] } }, "standard": { "lab": { "score": 47, "band": "conditional", "coverage": 8, "range": [ 47, 47 ] }, "regulator": { "score": 48, "band": "conditional", "coverage": 8, "range": [ 48, 48 ] }, "public": { "score": 48, "band": "conditional", "coverage": 8, "range": [ 48, 48 ] }, "equal": { "score": 47, "band": "conditional", "coverage": 8, "range": [ 47, 47 ] } }, "against_interest": { "lab": { "score": 39, "band": "conditional", "coverage": 5, "range": [ 24, 63 ] }, "regulator": { "score": 36, "band": "conditional", "coverage": 5, "range": [ 20, 65 ] }, "public": { "score": 42, "band": "conditional", "coverage": 5, "range": [ 28, 63 ] }, "equal": { "score": 40, "band": "conditional", "coverage": 5, "range": [ 25, 63 ] } }, "spans": { "lab": { "score": 47, "band": "conditional", "coverage": 7, "range": [ 42, 52 ] }, "regulator": { "score": 47, "band": "conditional", "coverage": 7, "range": [ 43, 53 ] }, "public": { "score": 47, "band": "conditional", "coverage": 7, "range": [ 40, 55 ] }, "equal": { "score": 46, "band": "conditional", "coverage": 7, "range": [ 41, 53 ] } }, "primary": { "lab": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "regulator": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "public": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "equal": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] } } }, "band": "conditional", "coverage": 8, "floor": "G", "what_moves": { "lab": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 52, "band": "conditional", "base": 47 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 49, "band": "conditional", "base": 47 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 50, "band": "conditional", "base": 47 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 52, "band": "conditional", "base": 47 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 51, "band": "conditional", "base": 47 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 51, "band": "conditional", "base": 47 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 49, "band": "conditional", "base": 47 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 48, "band": "conditional", "base": 47 } ], "regulator": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 51, "band": "conditional", "base": 48 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 50, "band": "conditional", "base": 48 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 50, "band": "conditional", "base": 48 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 53, "band": "conditional", "base": 48 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 53, "band": "conditional", "base": 48 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 51, "band": "conditional", "base": 48 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 50, "band": "conditional", "base": 48 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 48, "band": "conditional", "base": 48 } ], "public": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 54, "band": "conditional", "base": 48 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 51, "band": "conditional", "base": 48 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 51, "band": "conditional", "base": 48 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 49, "band": "conditional", "base": 48 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 50, "band": "conditional", "base": 48 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 53, "band": "conditional", "base": 48 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 49, "band": "conditional", "base": 48 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 49, "band": "conditional", "base": 48 } ], "equal": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 50, "band": "conditional", "base": 47 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 50, "band": "conditional", "base": 47 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 50, "band": "conditional", "base": 47 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 50, "band": "conditional", "base": 47 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 50, "band": "conditional", "base": 47 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 50, "band": "conditional", "base": 47 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 50, "band": "conditional", "base": 47 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 50, "band": "conditional", "base": 47 } ] } }, { "id": "apollo", "name": "Apollo Research", "type": "pbc", "hq": "London, UK / San Francisco, US", "domains": [ "scheming", "autonomy", "assurance" ], "confidence": "high", "summary": "Scheming and deception specialist; ran evaluations for every major lab; converted to a public benefit corporation in January 2026.", "what_would_move_the_score": "Disclose revenue concentration by client and separate the product business from the evaluation practice.", "role": "referee", "dissent": { "lower": "Funding could be 1. Apollo asks the parties it evaluates to pay fair market value, says it ran evaluations for all major labs in 2025, and publishes no revenue split; its 2024 self-report named only about $2M of philanthropy, and its 2026 seed round included Macroscopic Ventures, an Anthropic Series A and B investor. If lab fees are the principal revenue, F.5 caps at 1, and a material lab-investor stake would cap at 1 under F.4.", "higher": "Funding could be 3. Apollo's norms bar outcome-contingent pay and side grants or investments from evaluated labs; the PBC post says it remains partly philanthropy-supported, and the index records about $4.4M from Coefficient and $1.1M from SFF. A published cap holding lab revenue at or below 10% of annual revenue, with market rates and publication rights, would lift the F.3 cap to 3, the anchor for mostly philanthropic money with some lab-linked pooled funds." }, "list_group": "referee", "graph_id": "evaluator#204", "values": { "F": 2, "G": 2, "P": 2, "A": 2, "S": 3, "R": 2, "M": 2, "X": 2 }, "values_by_policy": { "leads": { "F": 2, "G": 2, "P": 2, "A": 2, "S": 3, "R": 2, "M": 2, "X": 2 }, "standard": { "F": 2, "G": 2, "P": 2, "A": 2, "S": 3, "R": 2, "M": 2, "X": 2 }, "against_interest": { "F": 2, "G": 2, "P": null, "A": null, "S": 3, "R": null, "M": null, "X": 2 }, "spans": { "F": 2, "G": 2, "P": 2, "A": 2, "S": 3, "R": 2, "M": 2, "X": 2 }, "primary": { "F": null, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "apollo", "dimension": "F", "value": 2, "anchor": 2, "signals": [ "apollo.02", "apollo.11", "apollo.12", "apollo.15" ], "assessed": "2026-09-15", "assessor": "yohei/claude v1.0", "open_questions": [ "Reconcile the self-reported $1.5M with the index total of $4.41M by date." ], "graph_id": "assessment#595", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 2, "b": [ "apollo.02", "apollo.15" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "apollo.02", "apollo.15" ], "h": false, "c": false, "u": false, "x": [ "apollo.11" ] }, "against_interest": { "v": 2, "b": [ "apollo.02", "apollo.15" ], "h": false, "c": false, "u": false, "x": [ "apollo.11", "apollo.12" ] }, "spans": { "v": 2, "b": [ "apollo.02", "apollo.15" ], "h": false, "c": false, "u": false, "x": [ "apollo.11" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "apollo.02", "apollo.11", "apollo.12", "apollo.15" ] } }, "rationale_derived": "Capped at 2 by apollo.02 (F.3): Requests fair-market-value compensation from the parties it evaluates; revenue depends on evaluated labs.; capped at 2 by apollo.15 (F.4): Macroscopic Ventures, an investor in Anthropic's Series A and B, participated in Apollo's January 2026 seed round; 50Y led and no stake sizes were disclosed.. Floors up to 2 from apollo.12 do not exceed the cap. Not counted under this policy: apollo.11 (no confirmed source (imported)).", "rationale_leads": "Capped at 2 by apollo.02 (F.3): Requests fair-market-value compensation from the parties it evaluates; revenue depends on evaluated labs.; capped at 2 by apollo.15 (F.4): Macroscopic Ventures, an investor in Anthropic's Series A and B, participated in Apollo's January 2026 seed round; 50Y led and no stake sizes were disclosed.. Floors up to 2 from apollo.12 do not exceed the cap." }, "G": { "evaluator": "apollo", "dimension": "G", "value": 2, "anchor": 2, "signals": [ "apollo.01", "apollo.03", "apollo.10", "apollo.13", "apollo.14" ], "rationale": "Anchor 2: for-profit PBC with a published COI policy and designated mission seats; investors include a lab investor. Previously mis-scored at 3 as if nonprofit.", "assessed": "2026-09-15", "assessor": "yohei/claude v1.0", "open_questions": [ "Round size and Macroscopic's share.", "Who holds the mission seats." ], "graph_id": "assessment#596", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "apollo.03", "apollo.10", "apollo.14" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "apollo.03", "apollo.10", "apollo.14" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "apollo.03", "apollo.10", "apollo.14" ], "h": false, "c": false, "u": false, "x": [ "apollo.01", "apollo.13" ] }, "spans": { "v": 2, "b": [ "apollo.03", "apollo.10", "apollo.14" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "apollo.01", "apollo.03", "apollo.10", "apollo.13", "apollo.14" ] } }, "rationale_derived": "Capped at 2 by apollo.03 (G.2): Converted from fiscally sponsored nonprofit to a public benefit corporation in January 2026.; capped at 2 by apollo.10 (G.3): January 2026 seed round led by 50Y with Juniper, Macroscopic Ventures (an Anthropic Series A and B investor), Common Metal, SAIF and individuals; amount not stated. Mission board seats designated as a governance safeguard.; capped at 2 by apollo.14 (G.3): Macroscopic Ventures, a participant in the seed round, lists Anthropic Series A and Series B among its investments.. Floors up to 2 from apollo.01, apollo.13 do not exceed the cap.", "rationale_leads": "Capped at 2 by apollo.03 (G.2): Converted from fiscally sponsored nonprofit to a public benefit corporation in January 2026.; capped at 2 by apollo.10 (G.3): January 2026 seed round led by 50Y with Juniper, Macroscopic Ventures (an Anthropic Series A and B investor), Common Metal, SAIF and individuals; amount not stated. Mission board seats designated as a governance safeguard.; capped at 2 by apollo.14 (G.3): Macroscopic Ventures, a participant in the seed round, lists Anthropic Series A and Series B among its investments.. Floors up to 2 from apollo.01, apollo.13 do not exceed the cap." }, "P": { "evaluator": "apollo", "dimension": "P", "value": 2, "anchor": 2, "signals": [ "apollo.05" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#597", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "apollo.05" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "apollo.05" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "apollo.05" ] }, "spans": { "v": 2, "b": [ "apollo.05" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "apollo.05" ] } }, "rationale_derived": "Floored at 2 by apollo.05 (P.5): Recusal rule for any individual with a financial interest in an evaluated organization.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by apollo.05 (P.5): Recusal rule for any individual with a financial interest in an evaluated organization.. No admissible signal caps it." }, "A": { "evaluator": "apollo", "dimension": "A", "value": 2, "anchor": 2, "signals": [ "apollo.06" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "lab-controlled", "graph_id": "assessment#598", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "apollo.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "apollo.06" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "apollo.06" ] }, "spans": { "v": 2, "b": [ "apollo.06" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "apollo.06" ] } }, "rationale_derived": "Floored at 2 by apollo.06 (A.1): Ran pre-deployment evaluations for all major labs; contracted by UK AISI for deception evaluations.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by apollo.06 (A.1): Ran pre-deployment evaluations for all major labs; contracted by UK AISI for deception evaluations.. No admissible signal caps it." }, "S": { "evaluator": "apollo", "dimension": "S", "value": 3, "anchor": 3, "signals": [ "apollo.07" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#599", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "apollo.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "apollo.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "apollo.07" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "apollo.07" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "apollo.07" ] } }, "rationale_derived": "Capped at 3 by apollo.07 (S.6): Co-authored anti-scheming research with OpenAI while acting as its external scheming evaluator..", "rationale_leads": "Capped at 3 by apollo.07 (S.6): Co-authored anti-scheming research with OpenAI while acting as its external scheming evaluator.." }, "R": { "evaluator": "apollo", "dimension": "R", "value": 2, "anchor": 2, "signals": [ "apollo.04" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "unknown", "graph_id": "assessment#600", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "apollo.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "apollo.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "apollo.04" ] }, "spans": { "v": 2, "b": [ "apollo.04" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "apollo.04" ] } }, "rationale_derived": "Floored at 2 by apollo.04 (R.7): Record of adverse findings published, including in-context scheming and evaluation awareness.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by apollo.04 (R.7): Record of adverse findings published, including in-context scheming and evaluation awareness.. No admissible signal caps it." }, "M": { "evaluator": "apollo", "dimension": "M", "value": 2, "anchor": 2, "signals": [ "apollo.09" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#601", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "apollo.09" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "apollo.09" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "apollo.09" ] }, "spans": { "v": 2, "b": [ "apollo.09" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "apollo.09" ] } }, "rationale_derived": "Floored at 2 by apollo.09 (M.2): Publishes evaluation methodology and papers; differential publishing policy documented.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by apollo.09 (M.2): Publishes evaluation methodology and papers; differential publishing policy documented.. No admissible signal caps it." }, "X": { "evaluator": "apollo", "dimension": "X", "value": 2, "anchor": 2, "signals": [ "apollo.08" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#602", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "apollo.08" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "apollo.08" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "apollo.08" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "apollo.08" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "apollo.08" ] } }, "rationale_derived": "Capped at 2 by apollo.08 (X.1): Sells Watcher, a monitoring product, into the ecosystem it evaluates..", "rationale_leads": "Capped at 2 by apollo.08 (X.1): Sells Watcher, a monitoring product, into the ecosystem it evaluates.." } }, "signals": [ { "id": "apollo.01", "evaluator": "apollo", "dimension": "G", "direction": "for", "claim": "Published four-rule COI policy: no outcome-contingent pay, no side grants or investments from evaluated labs, recusal for financial interests, no misrepresentation.", "sources": [ "apollo-norms" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "quote": "We do not accept work where compensation is contingent upon the outcome", "bound": { "floor": 2 }, "rule": "G.2", "quote_source": "apollo-norms", "graph_id": "signal#241", "source_graph_ids": [ "source#32" ] }, { "id": "apollo.02", "evaluator": "apollo", "dimension": "F", "direction": "against", "claim": "Requests fair-market-value compensation from the parties it evaluates; revenue depends on evaluated labs.", "sources": [ "apollo-norms" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "F.3", "quote": "we generally request that the parties for whom we conduct work compensate us at a fair market value for the work", "quote_source": "apollo-norms", "graph_id": "signal#242", "source_graph_ids": [ "source#32" ] }, { "id": "apollo.03", "evaluator": "apollo", "dimension": "G", "direction": "against", "claim": "Converted from fiscally sponsored nonprofit to a public benefit corporation in January 2026.", "sources": [ "apollo-pbc" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "G.2", "quote": "Apollo is spinning off from our fiscal sponsor into a Public Benefit Corporation (PBC)", "quote_source": "apollo-pbc", "graph_id": "signal#243", "source_graph_ids": [ "source#35" ] }, { "id": "apollo.04", "evaluator": "apollo", "dimension": "R", "direction": "for", "claim": "Record of adverse findings published, including in-context scheming and evaluation awareness.", "sources": [ "apollo-about" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "R.7", "quote": "published the first evidence that frontier models can scheme in context", "quote_source": "apollo-about", "graph_id": "signal#244", "source_graph_ids": [ "source#29" ] }, { "id": "apollo.05", "evaluator": "apollo", "dimension": "P", "direction": "for", "claim": "Recusal rule for any individual with a financial interest in an evaluated organization.", "sources": [ "apollo-norms" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "P.5", "quote": "We recuse any individual who is considered to have a financial interest in an organization we are evaluating", "quote_source": "apollo-norms", "graph_id": "signal#245", "source_graph_ids": [ "source#32" ] }, { "id": "apollo.06", "evaluator": "apollo", "dimension": "A", "direction": "for", "claim": "Ran pre-deployment evaluations for all major labs; contracted by UK AISI for deception evaluations.", "sources": [ "apollo-about", "apollo-first-year" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "A.1", "quote": "partnered with OpenAI to test their o1 model before public deployment", "quote_source": "apollo-about", "graph_id": "signal#246", "source_graph_ids": [ "source#29", "source#30" ] }, { "id": "apollo.07", "evaluator": "apollo", "dimension": "S", "direction": "against", "claim": "Co-authored anti-scheming research with OpenAI while acting as its external scheming evaluator.", "sources": [ "apollo-about" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 3 }, "rule": "S.6", "quote": "partnered with OpenAI to study anti-scheming interventions on frontier models", "quote_source": "apollo-about", "graph_id": "signal#247", "source_graph_ids": [ "source#29" ] }, { "id": "apollo.08", "evaluator": "apollo", "dimension": "X", "direction": "against", "claim": "Sells Watcher, a monitoring product, into the ecosystem it evaluates.", "sources": [ "apollo-about" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "X.1", "quote": "building an AI security tool to monitor frontier AI agents", "quote_source": "apollo-about", "graph_id": "signal#248", "source_graph_ids": [ "source#29" ] }, { "id": "apollo.09", "evaluator": "apollo", "dimension": "M", "direction": "for", "claim": "Publishes evaluation methodology and papers; differential publishing policy documented.", "sources": [ "apollo-norms" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "M.2", "quote": "We engage in differential publishing", "quote_source": "apollo-norms", "graph_id": "signal#249", "source_graph_ids": [ "source#32" ] }, { "id": "apollo.10", "evaluator": "apollo", "dimension": "G", "direction": "against", "claim": "January 2026 seed round led by 50Y with Juniper, Macroscopic Ventures (an Anthropic Series A and B investor), Common Metal, SAIF and individuals; amount not stated. Mission board seats designated as a governance safeguard.", "sources": [ "apollo-pbc-round" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "quote": "raised a Seed round led by 50Y", "bound": { "cap": 2 }, "rule": "G.3", "quote_source": "apollo-pbc-round", "graph_id": "signal#250", "source_graph_ids": [ "source#34" ] }, { "id": "apollo.11", "evaluator": "apollo", "dimension": "F", "direction": "against", "claim": "SFF recommendations of about $1.1M and Coefficient grants of about $4.4M per the index snapshot; both funders are one to two steps from an Anthropic investor.", "sources": [ "evaluators-ledger", "coefficient-index" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-11", "bound": { "cap": 3 }, "rule": "F.6", "graph_id": "signal#251", "source_graph_ids": [ "source#81", "source#52" ] }, { "id": "apollo.12", "evaluator": "apollo", "dimension": "F", "direction": "for", "claim": "Self-reported philanthropic base: about $1.5M from Open Philanthropy and about $500K from SFF at the time of its Manifund listing; a later index total of $4.41M is recorded as an imported row and the two figures are not reconciled.", "sources": [ "apollo-manifund", "evaluators-ledger" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2026-09-15", "quote": "We have received ~$1.5M from OpenPhil and ~$500k from SFF", "bound": { "floor": 2 }, "rule": "F.5", "quote_source": "apollo-manifund", "graph_id": "signal#252", "source_graph_ids": [ "source#31", "source#81" ] }, { "id": "apollo.13", "evaluator": "apollo", "dimension": "G", "direction": "for", "claim": "Mission seats on the PBC board are held by directors independent of the company and its funders, with a mandate to prioritize mission.", "sources": [ "apollo-pbc-mission" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2026-09-15", "quote": "who are independent to Apollo Research PBC and our funders", "bound": { "floor": 2 }, "rule": "G.2", "quote_source": "apollo-pbc-mission", "graph_id": "signal#253", "source_graph_ids": [ "source#33" ] }, { "id": "apollo.14", "evaluator": "apollo", "dimension": "G", "direction": "against", "claim": "Macroscopic Ventures, a participant in the seed round, lists Anthropic Series A and Series B among its investments.", "sources": [ "premieralts-macroscopic", "apollo-pbc-round" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2026", "bound": { "cap": 2 }, "rule": "G.3", "quote": "further investments from Juniper Ventures , Macroscopic Ventures", "quote_source": "apollo-pbc-round", "graph_id": "signal#254", "source_graph_ids": [ "source#151", "source#34" ] }, { "id": "apollo.15", "evaluator": "apollo", "dimension": "F", "direction": "against", "claim": "Macroscopic Ventures, an investor in Anthropic's Series A and B, participated in Apollo's January 2026 seed round; 50Y led and no stake sizes were disclosed.", "sources": [ "apollo-pbc-round", "premieralts-macroscopic" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 2 }, "rule": "F.4", "quote": "further investments from Juniper Ventures , Macroscopic Ventures", "quote_source": "apollo-pbc-round", "note": "F.4 caps at 1 only for a material investment (a priced round the fund led or co-led, or a stake of 5% or more); Macroscopic did not lead and its share is undisclosed, so the cap stays at 2 (equity from a lab investor bars anchor 3). Document request: Macroscopic's share of the round.", "graph_id": "signal#255", "source_graph_ids": [ "source#34", "source#151" ] } ], "scores": { "lab": 54, "regulator": 55, "public": 53, "equal": 53 }, "scores_by_policy": { "leads": { "lab": { "score": 54, "band": "clear", "coverage": 8, "range": [ 54, 54 ] }, "regulator": { "score": 55, "band": "clear", "coverage": 8, "range": [ 55, 55 ] }, "public": { "score": 53, "band": "clear", "coverage": 8, "range": [ 53, 53 ] }, "equal": { "score": 53, "band": "clear", "coverage": 8, "range": [ 53, 53 ] } }, "standard": { "lab": { "score": 54, "band": "clear", "coverage": 8, "range": [ 54, 54 ] }, "regulator": { "score": 55, "band": "clear", "coverage": 8, "range": [ 55, 55 ] }, "public": { "score": 53, "band": "clear", "coverage": 8, "range": [ 53, 53 ] }, "equal": { "score": 53, "band": "clear", "coverage": 8, "range": [ 53, 53 ] } }, "against_interest": { "lab": { "score": 59, "band": "clear", "coverage": 4, "range": [ 28, 81 ] }, "regulator": { "score": 61, "band": "clear", "coverage": 4, "range": [ 28, 83 ] }, "public": { "score": 55, "band": "clear", "coverage": 4, "range": [ 30, 75 ] }, "equal": { "score": 56, "band": "clear", "coverage": 4, "range": [ 28, 78 ] } }, "spans": { "lab": { "score": 54, "band": "clear", "coverage": 8, "range": [ 54, 54 ] }, "regulator": { "score": 55, "band": "clear", "coverage": 8, "range": [ 55, 55 ] }, "public": { "score": 53, "band": "clear", "coverage": 8, "range": [ 53, 53 ] }, "equal": { "score": 53, "band": "clear", "coverage": 8, "range": [ 53, 53 ] } }, "primary": { "lab": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "regulator": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "public": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "equal": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] } } }, "band": "clear", "coverage": 8, "floor": "F", "what_moves": { "lab": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 59, "band": "clear", "base": 54 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 56, "band": "clear", "base": 54 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 56, "band": "clear", "base": 54 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 59, "band": "clear", "base": 54 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 58, "band": "clear", "base": 54 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 57, "band": "clear", "base": 54 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 56, "band": "clear", "base": 54 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 55, "band": "clear", "base": 54 } ], "regulator": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 59, "band": "clear", "base": 55 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 57, "band": "clear", "base": 55 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 57, "band": "clear", "base": 55 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 60, "band": "clear", "base": 55 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 60, "band": "clear", "base": 55 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 59, "band": "clear", "base": 55 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 57, "band": "clear", "base": 55 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 55, "band": "clear", "base": 55 } ], "public": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 59, "band": "clear", "base": 53 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 56, "band": "clear", "base": 53 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 56, "band": "clear", "base": 53 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 54, "band": "clear", "base": 53 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 55, "band": "clear", "base": 53 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 57, "band": "clear", "base": 53 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 54, "band": "clear", "base": 53 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 54, "band": "clear", "base": 53 } ], "equal": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 56, "band": "clear", "base": 53 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 56, "band": "clear", "base": 53 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 56, "band": "clear", "base": 53 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 56, "band": "clear", "base": 53 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 56, "band": "clear", "base": 53 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 56, "band": "clear", "base": 53 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 56, "band": "clear", "base": 53 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 56, "band": "clear", "base": 53 } ] } }, { "id": "averi", "name": "AVERI", "type": "nonprofit", "hq": "Washington, US", "domains": [ "assurance" ], "confidence": "med", "summary": "Standards body for frontier AI auditing; ran the first double-blind enclave evaluation of a proprietary model.", "what_would_move_the_score": "Funder disclosure and a first audit under statutory terms.", "role": "referee", "dissent": { "lower": "Access is at the floor; no lower notch exists. The public record supports 0: AVERI's only completed engagement queried Gemini 2.5 Flash-Lite, a model public since July 2025, inside an enclave that hid the weights from the auditors, and the report says the pilot addressed some but not all ways a developer could tamper with the evaluation. No pre-release checkpoint, safeguards-off run, log or on-site access appears anywhere on the record.", "higher": "Access could be 1. AVERI reports API credits from six labs \"as donations or compensation\" and says it is pursuing pilots on whether companies follow their own safety and security frameworks; Google DeepMind ran a proprietary model for it inside a trusted execution environment it did not control. A nonpublic endpoint or pre-release checkpoint in the current pilots would meet anchor 1, and the enclave protocol is a documented route to deeper access." }, "list_group": "referee", "graph_id": "evaluator#205", "values": { "F": 3, "G": 3, "P": 3, "A": 0, "S": 3, "R": 2, "M": 3, "X": 3 }, "values_by_policy": { "leads": { "F": 3, "G": 3, "P": 3, "A": 0, "S": 3, "R": 2, "M": 3, "X": 3 }, "standard": { "F": 3, "G": 3, "P": 3, "A": 0, "S": 3, "R": 2, "M": 3, "X": 3 }, "against_interest": { "F": 3, "G": null, "P": 3, "A": 0, "S": 3, "R": 2, "M": 3, "X": null }, "spans": { "F": 3, "G": 3, "P": 3, "A": 0, "S": 3, "R": 2, "M": 3, "X": 3 }, "primary": { "F": null, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "averi", "dimension": "F", "value": 3, "anchor": 3, "signals": [ "averi.06", "averi.09", "averi.11", "averi.12", "averi.14", "averi.17" ], "rationale": "Anchor 3: philanthropic and diversified with named funders and a no-majority rule; lab-employee donations and two lab-adjacent funders keep it off anchor 4.", "assessed": "2026-09-15", "assessor": "yohei/claude v1.0", "open_questions": [ "Dollar shares per funder." ], "graph_id": "assessment#603", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "averi.06", "averi.11", "averi.14" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "averi.06", "averi.11", "averi.14" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "averi.06", "averi.11", "averi.14" ], "h": false, "c": false, "u": false, "x": [ "averi.09" ] }, "spans": { "v": 3, "b": [ "averi.06", "averi.11", "averi.14" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "averi.06", "averi.09", "averi.11", "averi.12", "averi.14", "averi.17" ] } }, "rationale_derived": "Capped at 3 by averi.06 (F.12): Funding sources not fully itemized publicly.; capped at 3 by averi.11 (F.6): AIUC, a funder, was seeded by an Anthropic co-founder; Halcyon's CEO chairs Transluce's board; Coefficient's principal is an Anthropic investor. Two of the named funders are two steps from labs.; capped at 3 by averi.14 (F.7): AVERI accepts and has received donations from current and former non-executive employees and alumni of frontier AI companies; amounts are not disclosed.. Floors up to 3 from averi.09, averi.12 do not exceed the cap.", "rationale_leads": "Capped at 3 by averi.06 (F.12): Funding sources not fully itemized publicly.; capped at 3 by averi.11 (F.6): AIUC, a funder, was seeded by an Anthropic co-founder; Halcyon's CEO chairs Transluce's board; Coefficient's principal is an Anthropic investor. Two of the named funders are two steps from labs.; capped at 3 by averi.14 (F.7): AVERI accepts and has received donations from current and former non-executive employees and alumni of frontier AI companies; amounts are not disclosed.. Floors up to 3 from averi.09, averi.12 do not exceed the cap." }, "G": { "evaluator": "averi", "dimension": "G", "value": 3, "anchor": 3, "signals": [ "averi.02", "averi.13" ], "rationale": "Anchor 3: 501(c)(3) nonprofit (averi.02). No tier-1 source for an independent board or external review, so 4 is unearned under the tier rule (C10).", "assessed": "2026-09-15", "assessor": "yohei/claude v0", "graph_id": "assessment#604", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "averi.13" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "averi.13" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "averi.02", "averi.13" ] }, "spans": { "v": 3, "b": [ "averi.13" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "averi.02", "averi.13" ] } }, "rationale_derived": "Floored at 3 by averi.13 (G.2): Published conflict-of-interest rules: all employees, board members and contractors disclose conflicts on starting and biannually; material conflicts trigger mandatory recusal or declining the engagement; no cash donations or compensation are accepted from frontier AI companies or their executives; lab API credits and lab-employee donations are disclosed; the most significant conflicts are disclosed publicly.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by averi.13 (G.2): Published conflict-of-interest rules: all employees, board members and contractors disclose conflicts on starting and biannually; material conflicts trigger mandatory recusal or declining the engagement; no cash donations or compensation are accepted from frontier AI companies or their executives; lab API credits and lab-employee donations are disclosed; the most significant conflicts are disclosed publicly.. No admissible signal caps it." }, "P": { "evaluator": "averi", "dimension": "P", "value": 3, "anchor": 3, "signals": [ "averi.04", "averi.10" ], "rationale": "Anchor 3: recusal policy with a stated cooling-off for the director and disclosed ties; short of anchor 4 because the cooling-off is person-specific rather than a standing rule.", "assessed": "2026-09-14", "assessor": "yohei/claude v0.4", "graph_id": "assessment#605", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "averi.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "averi.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "averi.04" ], "h": false, "c": false, "u": false, "x": [ "averi.10" ] }, "spans": { "v": 3, "b": [ "averi.04" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "averi.04", "averi.10" ] } }, "rationale_derived": "Capped at 3 by averi.04 (P.3): Founder and several staff come from labs and lab-adjacent institutions.. Floors up to 3 from averi.10 do not exceed the cap.", "rationale_leads": "Capped at 3 by averi.04 (P.3): Founder and several staff come from labs and lab-adjacent institutions.. Floors up to 3 from averi.10 do not exceed the cap." }, "A": { "evaluator": "averi", "dimension": "A", "value": 0, "anchor": 0, "signals": [ "averi.01", "averi.16" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "lab-controlled", "graph_id": "assessment#606", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 0, "b": [ "averi.16" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 0, "b": [ "averi.16" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 0, "b": [ "averi.16" ], "h": false, "c": false, "u": false, "x": [ "averi.01" ] }, "spans": { "v": 0, "b": [ "averi.16" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "averi.01", "averi.16" ] } }, "rationale_derived": "Capped at 0 by averi.16 (A.1): AVERI's only completed engagement evaluated Gemini 2.5 Flash-Lite, a publicly released model, inside a secure enclave that hid the weights; no pre-release, safeguards-off, helpful-only, log or on-site access has been documented.. Floors up to 0 from averi.01 do not exceed the cap.", "rationale_leads": "Capped at 0 by averi.16 (A.1): AVERI's only completed engagement evaluated Gemini 2.5 Flash-Lite, a publicly released model, inside a secure enclave that hid the weights; no pre-release, safeguards-off, helpful-only, log or on-site access has been documented.. Floors up to 0 from averi.01 do not exceed the cap." }, "S": { "evaluator": "averi", "dimension": "S", "value": 3, "anchor": 3, "signals": [ "averi.05" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#607", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "averi.05" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "averi.05" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "averi.05" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "averi.05" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "averi.05" ] } }, "rationale_derived": "Capped at 3 by averi.05 (S.4): Pilot audits are voluntary and lab-selected; not yet a full-time auditor..", "rationale_leads": "Capped at 3 by averi.05 (S.4): Pilot audits are voluntary and lab-selected; not yet a full-time auditor.." }, "R": { "evaluator": "averi", "dimension": "R", "value": 2, "anchor": 2, "signals": [ "averi.07", "averi.15" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "lab-controlled", "graph_id": "assessment#608", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "averi.15" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "averi.15" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "averi.15" ], "h": false, "c": false, "u": false, "x": [ "averi.07" ] }, "spans": { "v": 2, "b": [ "averi.15" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "averi.07", "averi.15" ] } }, "rationale_derived": "Capped at 2 by averi.15 (R.2): The findings of the only completed pilot were delivered to Google DeepMind as a confidential report; the public pilot report describes the protocol and the types of successes and failure modes without quantitative results, prompts or outputs.. Floors up to 2 from averi.07 do not exceed the cap.", "rationale_leads": "Capped at 2 by averi.15 (R.2): The findings of the only completed pilot were delivered to Google DeepMind as a confidential report; the public pilot report describes the protocol and the types of successes and failure modes without quantitative results, prompts or outputs.. Floors up to 2 from averi.07 do not exceed the cap." }, "M": { "evaluator": "averi", "dimension": "M", "value": 3, "anchor": 3, "signals": [ "averi.03" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#609", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "averi.03" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "averi.03" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "averi.03" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "averi.03" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "averi.03" ] } }, "rationale_derived": "Floored at 3 by averi.03 (M.1): Publishes pilot reports and open tooling; framework paper with AI Assurance Levels.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by averi.03 (M.1): Publishes pilot reports and open tooling; framework paper with AI Assurance Levels.. No admissible signal caps it." }, "X": { "evaluator": "averi", "dimension": "X", "value": 3, "anchor": 3, "signals": [ "averi.08" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#610", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "averi.08" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "averi.08" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "averi.08" ] }, "spans": { "v": 3, "b": [ "averi.08" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "averi.08" ] } }, "rationale_derived": "Floored at 3 by averi.08 (X.4): No commercial products; tools open source.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by averi.08 (X.4): No commercial products; tools open source.. No admissible signal caps it." } }, "signals": [ { "id": "averi.01", "evaluator": "averi", "dimension": "A", "direction": "for", "claim": "Secure-enclave double-blind protocol with DeepMind, OpenMined and MLCommons; neither party sees the other's confidential material.", "sources": [ "averi-pilot" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 0 }, "rule": "A.1", "quote": "The pilot tested Gemini 2.5 Flash-Lite using never-before-used prompts from the MLCommons safety benchmark family", "quote_source": "averi-pilot", "graph_id": "signal#256", "source_graph_ids": [ "source#42" ] }, { "id": "averi.02", "evaluator": "averi", "dimension": "G", "direction": "for", "claim": "501(c)(3); endorsed Illinois SB 315; co-authored AEF-1.", "sources": [ "averi-launch", "averi-about" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "G.1", "quote": "AVERI is a 501(c)(3), US-based non-profit think tank", "quote_source": "averi-launch", "graph_id": "signal#257", "source_graph_ids": [ "source#41", "source#39" ] }, { "id": "averi.03", "evaluator": "averi", "dimension": "M", "direction": "for", "claim": "Publishes pilot reports and open tooling; framework paper with AI Assurance Levels.", "sources": [ "averi-pilot", "arxiv-frontier-auditing" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "M.1", "quote": "converts what we learn into auditing standards, policy analysis, and open source tools", "quote_source": "averi-pilot", "graph_id": "signal#258", "source_graph_ids": [ "source#42", "source#37" ] }, { "id": "averi.04", "evaluator": "averi", "dimension": "P", "direction": "against", "claim": "Founder and several staff come from labs and lab-adjacent institutions.", "sources": [ "averi-launch" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 3 }, "rule": "P.3", "quote": "I hope that AVERI will also benefit from the practical experiences I had at OpenAI", "quote_source": "averi-launch", "graph_id": "signal#259", "source_graph_ids": [ "source#41" ] }, { "id": "averi.05", "evaluator": "averi", "dimension": "S", "direction": "against", "claim": "Pilot audits are voluntary and lab-selected; not yet a full-time auditor.", "sources": [ "averi-about", "averi-pilot" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 3 }, "rule": "S.4", "quote": "some companies have chosen to conduct voluntary pilots with AVERI and other organizations", "quote_source": "averi-about", "graph_id": "signal#260", "source_graph_ids": [ "source#39", "source#42" ] }, { "id": "averi.06", "evaluator": "averi", "dimension": "F", "direction": "against", "claim": "Funding sources not fully itemized publicly.", "sources": [ "averi-about" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 3 }, "rule": "F.12", "quote": "Our funders include the AI Underwriting Company, Coefficient Giving, Craig Falls, Geoff Ralston, Good Forever Foundation", "quote_source": "averi-about", "graph_id": "signal#261", "source_graph_ids": [ "source#39" ] }, { "id": "averi.07", "evaluator": "averi", "dimension": "R", "direction": "for", "claim": "Publishes pilot findings.", "sources": [ "averi-pilot" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "R.2", "quote": "We provided Google DeepMind with a confidential report of the findings", "quote_source": "averi-pilot", "graph_id": "signal#262", "source_graph_ids": [ "source#42" ] }, { "id": "averi.08", "evaluator": "averi", "dimension": "X", "direction": "for", "claim": "No commercial products; tools open source.", "sources": [ "averi-about" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "X.4", "quote": "We convert insights from these processes into open source tools", "quote_source": "averi-about", "graph_id": "signal#263", "source_graph_ids": [ "source#39" ] }, { "id": "averi.09", "evaluator": "averi", "dimension": "F", "direction": "for", "claim": "Funders disclosed by name (AIUC, Coefficient, Halcyon, Fathom, Sympatico Ventures, individuals) with no donor a majority; donations from non-executive lab employees disclosed; API credits offered by six labs and disclosed.", "sources": [ "averi-funding" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "quote": "No donor makes up a majority of our funding", "bound": { "floor": 3 }, "rule": "F.12", "quote_source": "averi-funding", "graph_id": "signal#264", "source_graph_ids": [ "source#40" ] }, { "id": "averi.10", "evaluator": "averi", "dimension": "P", "direction": "for", "claim": "Executive director recused from auditing OpenAI for at least two years after leaving and sold eligible shares; team members' individual holdings disclosed; mandatory recusal for material conflicts.", "sources": [ "averi-funding" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "quote": "Miles is recused from directly auditing OpenAI until two years after his October 2024 departure", "bound": { "floor": 3 }, "rule": "P.5", "quote_source": "averi-funding", "graph_id": "signal#265", "source_graph_ids": [ "source#40" ] }, { "id": "averi.11", "evaluator": "averi", "dimension": "F", "direction": "against", "claim": "AIUC, a funder, was seeded by an Anthropic co-founder; Halcyon's CEO chairs Transluce's board; Coefficient's principal is an Anthropic investor. Two of the named funders are two steps from labs.", "sources": [ "averi-funding", "apollo-pbc-round" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 3 }, "rule": "F.6", "quote": "Our funders include the AI Underwriting Company, Coefficient Giving", "quote_source": "averi-funding", "graph_id": "signal#266", "source_graph_ids": [ "source#40", "source#34" ] }, { "id": "averi.12", "evaluator": "averi", "dimension": "F", "direction": "for", "claim": "Raised $7.5 million toward a $13 million goal at launch; funders named; a search of Coefficient's grants pages found no discrete AVERI grant page though AVERI names Coefficient as a funder.", "sources": [ "fortune-averi", "averi-funding" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2026-01-15", "quote": "AVERI said it has raised $7.5 million toward a goal of $13 million", "bound": { "floor": 3 }, "rule": "F.12", "quote_source": "fortune-averi", "graph_id": "signal#267", "source_graph_ids": [ "source#86", "source#40" ] }, { "id": "averi.13", "evaluator": "averi", "dimension": "G", "direction": "for", "claim": "Published conflict-of-interest rules: all employees, board members and contractors disclose conflicts on starting and biannually; material conflicts trigger mandatory recusal or declining the engagement; no cash donations or compensation are accepted from frontier AI companies or their executives; lab API credits and lab-employee donations are disclosed; the most significant conflicts are disclosed publicly.", "sources": [ "averi-funding" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "floor": 3 }, "rule": "G.2", "quote": "mitigation steps may include recusing personnel from projects or declining an engagement entirely", "quote_source": "averi-funding", "note": "G.2: a nonprofit with a published policy naming at least two of recusal for financial interests, no side money from evaluated labs, and disclosure of lab-linked revenue floors at 3. G.7 (4) needs tier-1 evidence of an independent board and external review.", "graph_id": "signal#268", "source_graph_ids": [ "source#40" ] }, { "id": "averi.14", "evaluator": "averi", "dimension": "F", "direction": "against", "claim": "AVERI accepts and has received donations from current and former non-executive employees and alumni of frontier AI companies; amounts are not disclosed.", "sources": [ "averi-funding", "fortune-averi" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "F.7", "quote": "several non-executive employees and alumni of frontier AI companies", "quote_source": "averi-funding", "note": "F.7: undisclosed amounts carry no bound, but a policy that permits lab-employee donations caps at 3. Fortune corroborates (\"donations from current and former non-executive employees of frontier AI companies\").", "graph_id": "signal#269", "source_graph_ids": [ "source#40", "source#86" ] }, { "id": "averi.15", "evaluator": "averi", "dimension": "R", "direction": "against", "claim": "The findings of the only completed pilot were delivered to Google DeepMind as a confidential report; the public pilot report describes the protocol and the types of successes and failure modes without quantitative results, prompts or outputs.", "sources": [ "averi-pilot" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 2 }, "rule": "R.2", "quote": "We provided Google DeepMind with a confidential report of the findings", "quote_source": "averi-pilot", "note": "Anchor 2 at most: publication with the findings withheld. No rule names this case (R.2 covers editorial feedback, R.3 system-card-only); recorded as a gap and capped by the anchor text.", "graph_id": "signal#270", "source_graph_ids": [ "source#42" ] }, { "id": "averi.16", "evaluator": "averi", "dimension": "A", "direction": "against", "claim": "AVERI's only completed engagement evaluated Gemini 2.5 Flash-Lite, a publicly released model, inside a secure enclave that hid the weights; no pre-release, safeguards-off, helpful-only, log or on-site access has been documented.", "sources": [ "averi-pilot", "averi-about" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 0 }, "rule": "A.1", "quote": "The pilot tested Gemini 2.5 Flash-Lite", "quote_source": "averi-pilot", "note": "A.1 scores what was granted: public-model access is anchor 0. C17: an against-signal from the organization's own page with a span is an admission against interest, so the 0 is effective. Flagged for review: the stored 3 read the enclave protocol as access depth.", "graph_id": "signal#271", "source_graph_ids": [ "source#42", "source#39" ] }, { "id": "averi.17", "evaluator": "averi", "dimension": "F", "direction": "against", "claim": "AVERI has received API credits \"as donations or compensation\" from Amazon, Anthropic, Google DeepMind, Microsoft, OpenAI and Thinking Machines Lab; amounts are undisclosed.", "sources": [ "averi-funding" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": null, "rule": "F.11", "quote": "We've received API credits as donations or compensation from Amazon, Anthropic, Google DeepMind, Microsoft, OpenAI", "quote_source": "averi-funding", "note": "F.11: undisclosed in-kind credits with no dependency statement carry no funding bound; recorded on the access mechanism. The page says credits came \"as donations or compensation\": credits received as compensation for pilot work would be per-engagement lab payment in kind (F.3, cap 2). Document request: which labs provided credits as compensation and for which engagement. Bound left null pending that document.", "bound_note": "F.11: undisclosed in-kind credits without a dependency statement set no funding bound; recorded on the access mechanism. Null pending a document on which credits were compensation for pilot work (F.3 would then cap at 2).", "graph_id": "signal#272", "source_graph_ids": [ "source#40" ] } ], "scores": { "lab": 56, "regulator": 56, "public": 66, "equal": 63 }, "scores_by_policy": { "leads": { "lab": { "score": 56, "band": "disqualifying", "coverage": 8, "range": [ 56, 56 ] }, "regulator": { "score": 56, "band": "disqualifying", "coverage": 8, "range": [ 56, 56 ] }, "public": { "score": 66, "band": "disqualifying", "coverage": 8, "range": [ 66, 66 ] }, "equal": { "score": 63, "band": "disqualifying", "coverage": 8, "range": [ 63, 63 ] } }, "standard": { "lab": { "score": 56, "band": "disqualifying", "coverage": 8, "range": [ 56, 56 ] }, "regulator": { "score": 56, "band": "disqualifying", "coverage": 8, "range": [ 56, 56 ] }, "public": { "score": 66, "band": "disqualifying", "coverage": 8, "range": [ 66, 66 ] }, "equal": { "score": 63, "band": "disqualifying", "coverage": 8, "range": [ 63, 63 ] } }, "against_interest": { "lab": { "score": 54, "band": "disqualifying", "coverage": 6, "range": [ 47, 60 ] }, "regulator": { "score": 54, "band": "disqualifying", "coverage": 6, "range": [ 49, 59 ] }, "public": { "score": 64, "band": "disqualifying", "coverage": 6, "range": [ 51, 71 ] }, "equal": { "score": 58, "band": "disqualifying", "coverage": 6, "range": [ 44, 69 ] } }, "spans": { "lab": { "score": 56, "band": "disqualifying", "coverage": 8, "range": [ 56, 56 ] }, "regulator": { "score": 56, "band": "disqualifying", "coverage": 8, "range": [ 56, 56 ] }, "public": { "score": 66, "band": "disqualifying", "coverage": 8, "range": [ 66, 66 ] }, "equal": { "score": 63, "band": "disqualifying", "coverage": 8, "range": [ 63, 63 ] } }, "primary": { "lab": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "regulator": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "public": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "equal": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] } } }, "band": "disqualifying", "coverage": 8, "floor": "A", "what_moves": { "lab": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 61, "band": "disqualifying", "base": 56 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 59, "band": "disqualifying", "base": 56 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 59, "band": "disqualifying", "base": 56 }, { "dimension": "A", "from_value": 0, "to_value": 1, "score": 62, "band": "conditional", "base": 56 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 61, "band": "disqualifying", "base": 56 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 60, "band": "disqualifying", "base": 56 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 59, "band": "disqualifying", "base": 56 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 58, "band": "disqualifying", "base": 56 } ], "regulator": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 60, "band": "disqualifying", "base": 56 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 59, "band": "disqualifying", "base": 56 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 59, "band": "disqualifying", "base": 56 }, { "dimension": "A", "from_value": 0, "to_value": 1, "score": 61, "band": "conditional", "base": 56 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 61, "band": "disqualifying", "base": 56 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 60, "band": "disqualifying", "base": 56 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 59, "band": "disqualifying", "base": 56 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 56, "band": "disqualifying", "base": 56 } ], "public": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 73, "band": "disqualifying", "base": 66 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 70, "band": "disqualifying", "base": 66 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 70, "band": "disqualifying", "base": 66 }, { "dimension": "A", "from_value": 0, "to_value": 1, "score": 68, "band": "conditional", "base": 66 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 69, "band": "disqualifying", "base": 66 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 71, "band": "disqualifying", "base": 66 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 68, "band": "disqualifying", "base": 66 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 68, "band": "disqualifying", "base": 66 } ], "equal": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 66, "band": "disqualifying", "base": 63 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 66, "band": "disqualifying", "base": 63 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 66, "band": "disqualifying", "base": 63 }, { "dimension": "A", "from_value": 0, "to_value": 1, "score": 66, "band": "conditional", "base": 63 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 66, "band": "disqualifying", "base": 63 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 66, "band": "disqualifying", "base": 63 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 66, "band": "disqualifying", "base": 63 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 66, "band": "disqualifying", "base": 63 } ] } }, { "id": "cais", "name": "Center for AI Safety", "type": "nonprofit", "hq": "San Francisco, US", "domains": [ "benchmarks", "misuse" ], "confidence": "med", "summary": "Hazard benchmarks (WMDP, HLE); director advises xAI.", "what_would_move_the_score": "A public recusal policy covering the xAI relationship.", "role": "referee", "dissent": { "lower": "Personnel should be 0. The director advises xAI while CAIS benchmarks (HLE, WMDP, MASK) are used to grade xAI models, and no recusal statement, disclosure page, or conflict policy is published anywhere on safe.ai. P.1 sets 0 for a governance role without recusal; an unrecused standing advisory role at a graded lab, with the benchmark scoring under the same director, is the nearest thing to it.", "higher": "Personnel should be 2. An advisory role is not a board or committee seat, so P.2's cap of 1 rather than P.1's 0 applies, and nothing above 1 was ever claimed. HLE and WMDP are scored mechanically on public leaderboards where xAI's results are visible to anyone, no equity, hiring flow, or second lab tie is documented, and a published recusal note would lift the value to 2." }, "list_group": "referee", "graph_id": "evaluator#206", "values": { "F": 3, "G": 2, "P": 1, "A": 1, "S": 3, "R": 2, "M": 3, "X": 3 }, "values_by_policy": { "leads": { "F": 3, "G": 2, "P": 1, "A": 1, "S": 3, "R": 2, "M": 3, "X": 3 }, "standard": { "F": 3, "G": 2, "P": 1, "A": 1, "S": 3, "R": 2, "M": 3, "X": 3 }, "against_interest": { "F": 3, "G": null, "P": 1, "A": 1, "S": null, "R": null, "M": null, "X": 3 }, "spans": { "F": 3, "G": 2, "P": 1, "A": null, "S": 3, "R": 2, "M": 3, "X": 3 }, "primary": { "F": null, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "cais", "dimension": "F", "value": 3, "anchor": 3, "signals": [ "cais.02", "cais.09" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#611", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "cais.09" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "cais.09" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "cais.09" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "cais.09" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "cais.02", "cais.09" ] } }, "rationale_derived": "Capped at 3 by cais.09 (F.6): SFF recommended $289,000 to CAIS in 2025 (about $1.1M cumulative per 80,000 Hours); SFF's funder Jaan Tallinn led Anthropic's Series A.. Floors up to 3 from cais.02 do not exceed the cap.", "rationale_leads": "Capped at 3 by cais.09 (F.6): SFF recommended $289,000 to CAIS in 2025 (about $1.1M cumulative per 80,000 Hours); SFF's funder Jaan Tallinn led Anthropic's Series A.. Floors up to 3 from cais.02 do not exceed the cap." }, "G": { "evaluator": "cais", "dimension": "G", "value": 2, "anchor": 2, "signals": [ "cais.05" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#612", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "cais.05" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "cais.05" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "cais.05" ] }, "spans": { "v": 2, "b": [ "cais.05" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "cais.05" ] } }, "rationale_derived": "Floored at 2 by cais.05 (G.1): Nonprofit.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by cais.05 (G.1): Nonprofit.. No admissible signal caps it." }, "P": { "evaluator": "cais", "dimension": "P", "value": 1, "anchor": 1, "signals": [ "cais.01" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "open_questions": [ "Request to the Center for AI Safety: the terms of the director's advisory role at xAI (start date, compensation or equity if any) and any recusal or disclosure statement covering CAIS benchmark results on xAI models." ], "graph_id": "assessment#613", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 1, "b": [ "cais.01" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "cais.01" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "cais.01" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "cais.01" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "cais.01" ] } }, "rationale_derived": "Capped at 1 by cais.01 (P.2): Director is a safety adviser to xAI while the organization's benchmarks are used to grade xAI models..", "rationale_leads": "Capped at 1 by cais.01 (P.2): Director is a safety adviser to xAI while the organization's benchmarks are used to grade xAI models.." }, "A": { "evaluator": "cais", "dimension": "A", "value": 1, "anchor": 1, "signals": [ "cais.06" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "lab-controlled", "evidence_limited": true, "graph_id": "assessment#614", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 1, "b": [ "cais.06" ], "h": true, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "cais.06" ], "h": true, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "cais.06" ], "h": true, "c": false, "u": false, "x": [] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "cais.06" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "cais.06" ] } }, "rationale_derived": "Capped at 1 by cais.06 (A.6): Public-model access.. Held: C15: a 0 needs a quoted span.", "rationale_leads": "Capped at 1 by cais.06 (A.6): Public-model access.. Held: C15: a 0 needs a quoted span." }, "S": { "evaluator": "cais", "dimension": "S", "value": 3, "anchor": 3, "signals": [ "cais.07" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#615", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "cais.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "cais.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "cais.07" ] }, "spans": { "v": 3, "b": [ "cais.07" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "cais.07" ] } }, "rationale_derived": "Floored at 3 by cais.07 (S.5): Sets own benchmark design.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by cais.07 (S.5): Sets own benchmark design.. No admissible signal caps it." }, "R": { "evaluator": "cais", "dimension": "R", "value": 2, "anchor": 2, "signals": [ "cais.08" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "self-imposed", "graph_id": "assessment#616", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "cais.08" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "cais.08" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "cais.08" ] }, "spans": { "v": 2, "b": [ "cais.08" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "cais.08" ] } }, "rationale_derived": "Floored at 2 by cais.08 (R.1): Publishes results.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by cais.08 (R.1): Publishes results.. No admissible signal caps it." }, "M": { "evaluator": "cais", "dimension": "M", "value": 3, "anchor": 3, "signals": [ "cais.03" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#617", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "cais.03" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "cais.03" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "cais.03" ] }, "spans": { "v": 3, "b": [ "cais.03" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "cais.03" ] } }, "rationale_derived": "Floored at 3 by cais.03 (M.3): Open benchmarks widely used.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by cais.03 (M.3): Open benchmarks widely used.. No admissible signal caps it." }, "X": { "evaluator": "cais", "dimension": "X", "value": 3, "anchor": 3, "signals": [ "cais.04" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#618", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "cais.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "cais.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "cais.04" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "cais.04" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "cais.04" ] } }, "rationale_derived": "Capped at 3 by cais.04 (X.6): Co-produced Humanity's Last Exam with Scale AI, a Meta-owned vendor..", "rationale_leads": "Capped at 3 by cais.04 (X.6): Co-produced Humanity's Last Exam with Scale AI, a Meta-owned vendor.." } }, "signals": [ { "id": "cais.01", "evaluator": "cais", "dimension": "P", "direction": "against", "claim": "Director is a safety adviser to xAI while the organization's benchmarks are used to grade xAI models.", "sources": [ "80k-funding", "cais" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 1 }, "rule": "P.2", "quote": "have driven some of the bigger successes in AI policy and advise xAI", "quote_source": "80k-funding", "graph_id": "signal#273", "source_graph_ids": [ "source#9", "source#44" ] }, { "id": "cais.02", "evaluator": "cais", "dimension": "F", "direction": "for", "claim": "Funded mainly by SFF rather than lab-linked pools.", "sources": [ "80k-funding" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "F.6", "quote": "SFF gave $1.1m to CAIS and $1.6m to the action fund", "quote_source": "80k-funding", "graph_id": "signal#274", "source_graph_ids": [ "source#9" ] }, { "id": "cais.03", "evaluator": "cais", "dimension": "M", "direction": "for", "claim": "Open benchmarks widely used.", "sources": [ "cais" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "M.3", "quote": "CAIS develops foundational benchmarks and methods which concretize the problem", "quote_source": "cais", "graph_id": "signal#275", "source_graph_ids": [ "source#44" ] }, { "id": "cais.04", "evaluator": "cais", "dimension": "X", "direction": "against", "claim": "Co-produced Humanity's Last Exam with Scale AI, a Meta-owned vendor.", "sources": [ "cais", "lastexam-hle", "cnbc-scale-meta" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 3 }, "rule": "X.6", "quote": "1 Center for AI Safety, 2 Scale AI", "quote_source": "lastexam-hle", "graph_id": "signal#276", "source_graph_ids": [ "source#44", "source#103", "source#50" ] }, { "id": "cais.05", "evaluator": "cais", "dimension": "G", "direction": "for", "claim": "Nonprofit.", "sources": [ "cais" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "G.1", "quote": "is a San Francisco-based research and field-building nonprofit", "quote_source": "cais", "graph_id": "signal#277", "source_graph_ids": [ "source#44" ] }, { "id": "cais.06", "evaluator": "cais", "dimension": "A", "direction": "against", "claim": "Public-model access.", "sources": [ "cais" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 0 }, "rule": "A.6", "graph_id": "signal#278", "source_graph_ids": [ "source#44" ] }, { "id": "cais.07", "evaluator": "cais", "dimension": "S", "direction": "for", "claim": "Sets own benchmark design.", "sources": [ "cais" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "S.5", "quote": "CAIS develops foundational benchmarks and methods which concretize the problem", "quote_source": "cais", "graph_id": "signal#279", "source_graph_ids": [ "source#44" ] }, { "id": "cais.08", "evaluator": "cais", "dimension": "R", "direction": "for", "claim": "Publishes results.", "sources": [ "cais" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "R.1", "quote": "The MASK Benchmark: Disentangling Honesty From Accuracy in AI Systems Mar 5, 2025", "quote_source": "cais", "graph_id": "signal#280", "source_graph_ids": [ "source#44" ] }, { "id": "cais.09", "evaluator": "cais", "dimension": "F", "direction": "against", "claim": "SFF recommended $289,000 to CAIS in 2025 (about $1.1M cumulative per 80,000 Hours); SFF's funder Jaan Tallinn led Anthropic's Series A.", "sources": [ "sff-2025", "anthropic-series-a", "80k-funding" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "F.6", "quote": "Center for AI Safety (CAIS) Main: $210,000 Freedom: $78,000 Fairness: $0 Mean: $0 $289,000", "quote_source": "sff-2025", "note": "Grants from a funder whose principal is an investor in a frontier developer cap at 3 (F.6). Ledger rows T57 and T71; anthropic-series-a: 'The Series A round was led by Jaan Tallinn'.", "graph_id": "signal#281", "source_graph_ids": [ "source#176", "source#28", "source#9" ] } ], "scores": { "lab": 55, "regulator": 54, "public": 56, "equal": 56 }, "scores_by_policy": { "leads": { "lab": { "score": 55, "band": "conditional", "coverage": 8, "range": [ 55, 55 ] }, "regulator": { "score": 54, "band": "conditional", "coverage": 8, "range": [ 54, 54 ] }, "public": { "score": 56, "band": "conditional", "coverage": 8, "range": [ 56, 56 ] }, "equal": { "score": 56, "band": "conditional", "coverage": 8, "range": [ 56, 56 ] } }, "standard": { "lab": { "score": 55, "band": "conditional", "coverage": 8, "range": [ 55, 55 ] }, "regulator": { "score": 54, "band": "conditional", "coverage": 8, "range": [ 54, 54 ] }, "public": { "score": 56, "band": "conditional", "coverage": 8, "range": [ 56, 56 ] }, "equal": { "score": 56, "band": "conditional", "coverage": 8, "range": [ 56, 56 ] } }, "against_interest": { "lab": { "score": 47, "band": "conditional", "coverage": 4, "range": [ 25, 72 ] }, "regulator": { "score": 42, "band": "conditional", "coverage": 4, "range": [ 19, 74 ] }, "public": { "score": 55, "band": "conditional", "coverage": 4, "range": [ 28, 78 ] }, "equal": { "score": 50, "band": "conditional", "coverage": 4, "range": [ 25, 75 ] } }, "spans": { "lab": { "score": 62, "band": "conditional", "coverage": 7, "range": [ 50, 70 ] }, "regulator": { "score": 61, "band": "conditional", "coverage": 7, "range": [ 49, 69 ] }, "public": { "score": 58, "band": "conditional", "coverage": 7, "range": [ 55, 60 ] }, "equal": { "score": 61, "band": "conditional", "coverage": 7, "range": [ 53, 66 ] } }, "primary": { "lab": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "regulator": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "public": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "equal": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] } } }, "band": "conditional", "coverage": 8, "floor": "P", "what_moves": { "lab": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 59, "band": "conditional", "base": 55 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 55 }, { "dimension": "P", "from_value": 1, "to_value": 2, "score": 57, "band": "conditional", "base": 55 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 60, "band": "conditional", "base": 55 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 59, "band": "conditional", "base": 55 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 58, "band": "conditional", "base": 55 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 57, "band": "conditional", "base": 55 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 56, "band": "conditional", "base": 55 } ], "regulator": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 57, "band": "conditional", "base": 54 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 54 }, { "dimension": "P", "from_value": 1, "to_value": 2, "score": 56, "band": "conditional", "base": 54 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 59, "band": "conditional", "base": 54 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 59, "band": "conditional", "base": 54 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 57, "band": "conditional", "base": 54 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 56, "band": "conditional", "base": 54 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 54, "band": "conditional", "base": 54 } ], "public": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 63, "band": "conditional", "base": 56 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 60, "band": "conditional", "base": 56 }, { "dimension": "P", "from_value": 1, "to_value": 2, "score": 60, "band": "conditional", "base": 56 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 57, "band": "conditional", "base": 56 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 61, "band": "conditional", "base": 56 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 57, "band": "conditional", "base": 56 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 57, "band": "conditional", "base": 56 } ], "equal": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "P", "from_value": 1, "to_value": 2, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 59, "band": "conditional", "base": 56 } ] } }, { "id": "caisi", "name": "US CAISI (NIST)", "type": "gov", "hq": "Gaithersburg, US", "domains": [ "cyber", "bio", "misuse", "assurance" ], "confidence": "high", "summary": "Pre-deployment agreements with five labs; 40+ evaluations; some classified via TRAINS taskforce.", "what_would_move_the_score": "Regular public reporting per model and a standing budget line insulated from reorganization.", "role": "government", "dissent": { "lower": "Publication (R) at 2 could be 1. For the five partner labs, no per-model evaluation has been published; the public write-ups concern a foreign model tested without cooperation and two vulnerability disclosures the labs had already fixed. CSA records \"no mandatory disclosure of findings\". Under R.4, a public body that publishes nothing per model for the labs it has agreements with scores 1, and the DeepSeek report is the exception rather than the practice.", "higher": "Publication (R) at 2 could be 3. CAISI has published a full per-model evaluation (DeepSeek V4 Pro), named vulnerabilities in ChatGPT Agent and Anthropic's Constitutional Classifiers, and states that it publishes findings from its assessments. No lab-edited summary or lab redaction is documented. That is closer to anchor 3 (publishes, redaction limited to security) than to the anchor-2 picture of broad lab redaction." }, "list_group": "government", "graph_id": "evaluator#207", "values": { "F": 3, "G": 3, "P": 3, "A": 3, "S": 3, "R": 2, "M": 2, "X": 3 }, "values_by_policy": { "leads": { "F": 3, "G": 3, "P": 3, "A": 3, "S": 3, "R": 2, "M": 2, "X": 3 }, "standard": { "F": 3, "G": 3, "P": 3, "A": 3, "S": 3, "R": null, "M": 2, "X": 3 }, "against_interest": { "F": 3, "G": 3, "P": 3, "A": 3, "S": 3, "R": null, "M": 2, "X": 3 }, "spans": { "F": 3, "G": 3, "P": 3, "A": 3, "S": 3, "R": null, "M": 2, "X": 3 }, "primary": { "F": null, "G": 3, "P": 3, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "caisi", "dimension": "F", "value": 3, "anchor": 3, "signals": [ "caisi.01", "caisi.10" ], "rationale": "Anchor 3 pending a confirmed bounded negative (appropriations text); access from labs is in-kind.", "assessed": "2026-09-14", "assessor": "yohei/claude v0.4", "open_questions": [ "Add a confirmed bounded negative from a primary source to restore anchor 4." ], "graph_id": "assessment#619", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "caisi.10" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "caisi.10" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "caisi.10" ], "h": false, "c": false, "u": false, "x": [ "caisi.01" ] }, "spans": { "v": 3, "b": [ "caisi.10" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "caisi.01", "caisi.10" ] } }, "rationale_derived": "Capped at 3 by caisi.10 (F.9): Public appropriation of about $10M in FY2026; free pre-deployment access from labs is in-kind. No confirmed primary-filing negative on lab money is on file.. Floors up to 3 from caisi.01 do not exceed the cap.", "rationale_leads": "Capped at 3 by caisi.10 (F.9): Public appropriation of about $10M in FY2026; free pre-deployment access from labs is in-kind. No confirmed primary-filing negative on lab money is on file.. Floors up to 3 from caisi.01 do not exceed the cap." }, "G": { "evaluator": "caisi", "dimension": "G", "value": 3, "anchor": 3, "signals": [ "caisi.03", "caisi.09", "caisi.11", "caisi.15" ], "assessed": "2026-09-15", "assessor": "yohei/claude v1.0", "graph_id": "assessment#620", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 3, "b": [ "caisi.03", "caisi.09", "caisi.11" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "caisi.09", "caisi.11" ], "h": false, "c": false, "u": false, "x": [ "caisi.03" ] }, "against_interest": { "v": 3, "b": [ "caisi.09", "caisi.11" ], "h": false, "c": false, "u": false, "x": [ "caisi.03" ] }, "spans": { "v": 3, "b": [ "caisi.09", "caisi.11" ], "h": false, "c": false, "u": false, "x": [ "caisi.03" ] }, "primary": { "v": 3, "b": [ "caisi.15" ], "h": false, "c": false, "u": false, "x": [ "caisi.03", "caisi.09", "caisi.11" ] } }, "rationale_derived": "Capped at 3 by caisi.09 (G.5): Appropriation of about $10M in FY2026 within NIST's AI line; the director resigned in July 2026 and the NIST director is acting head.; capped at 3 by caisi.11 (G.5): The CAISI director resigned on 20 July 2026, three months after appointment — the third leadership change in six months.. Floors up to 3 from caisi.15 do not exceed the cap. Not counted under this policy: caisi.03 (no confirmed source (unaudited)).", "rationale_leads": "Capped at 3 by caisi.03 (G.5): Refocused in 2025 from broad safety research to demonstrable national-security risks; mandate is politically steerable.; capped at 3 by caisi.09 (G.5): Appropriation of about $10M in FY2026 within NIST's AI line; the director resigned in July 2026 and the NIST director is acting head.; capped at 3 by caisi.11 (G.5): The CAISI director resigned on 20 July 2026, three months after appointment — the third leadership change in six months.. Floors up to 3 from caisi.15 do not exceed the cap." }, "P": { "evaluator": "caisi", "dimension": "P", "value": 3, "anchor": 3, "signals": [ "caisi.05", "caisi.14" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#621", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 3, "b": [ "caisi.14" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "caisi.14" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "caisi.14" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "caisi.14" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": 3, "b": [ "caisi.14" ], "h": false, "c": false, "u": false, "x": [ "caisi.05" ] } }, "rationale_derived": "Floored at 3 by caisi.14 (P.5): CAISI staff are NIST federal employees bound by the criminal conflict-of-interest statute (18 U.S.C. 208) and the post-employment statute (18 U.S.C. 207), which restricts former employees for one to two years after leaving.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by caisi.14 (P.5): CAISI staff are NIST federal employees bound by the criminal conflict-of-interest statute (18 U.S.C. 208) and the post-employment statute (18 U.S.C. 207), which restricts former employees for one to two years after leaving.. No admissible signal caps it." }, "A": { "evaluator": "caisi", "dimension": "A", "value": 3, "anchor": 3, "signals": [ "caisi.02", "caisi.12" ], "assessed": "2026-09-15", "assessor": "yohei/claude v0-final", "mechanism": "lab-controlled", "graph_id": "assessment#622", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 3, "b": [ "caisi.02", "caisi.12" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "caisi.12" ], "h": false, "c": false, "u": false, "x": [ "caisi.02" ] }, "against_interest": { "v": 3, "b": [ "caisi.12" ], "h": false, "c": false, "u": false, "x": [ "caisi.02" ] }, "spans": { "v": 3, "b": [ "caisi.12" ], "h": false, "c": false, "u": false, "x": [ "caisi.02" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "caisi.02", "caisi.12" ] } }, "rationale_derived": "Floored at 3 by caisi.12 (A.4): Five labs now give CAISI pre-deployment access (OpenAI and Anthropic since September 2025; Google DeepMind, Microsoft and xAI from 5 May 2026), including versions with guardrails stripped back; more than 40 evaluations completed, one of a foreign model without developer cooperation.. No admissible signal caps it. Not counted under this policy: caisi.02 (no confirmed source (unaudited)).", "rationale_leads": "Floored at 3 by caisi.02 (A.4): More than 40 evaluations including unreleased models; classified CBRN and cyber work via the TRAINS taskforce.; floored at 3 by caisi.12 (A.4): Five labs now give CAISI pre-deployment access (OpenAI and Anthropic since September 2025; Google DeepMind, Microsoft and xAI from 5 May 2026), including versions with guardrails stripped back; more than 40 evaluations completed, one of a foreign model without developer cooperation.. No admissible signal caps it." }, "S": { "evaluator": "caisi", "dimension": "S", "value": 3, "anchor": 3, "signals": [ "caisi.06", "caisi.13" ], "assessed": "2026-09-15", "assessor": "yohei/claude v0-final", "graph_id": "assessment#623", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "caisi.13" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "caisi.13" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "caisi.13" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "caisi.13" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "caisi.06", "caisi.13" ] } }, "rationale_derived": "Capped at 3 by caisi.13 (S.4): Participation is voluntary and agreements were jointly scoped with the two founding labs; the center can only assess models developers choose to share.. Floors up to 3 from caisi.06 do not exceed the cap.", "rationale_leads": "Capped at 3 by caisi.13 (S.4): Participation is voluntary and agreements were jointly scoped with the two founding labs; the center can only assess models developers choose to share.. Floors up to 3 from caisi.06 do not exceed the cap." }, "R": { "evaluator": "caisi", "dimension": "R", "value": 2, "anchor": 2, "signals": [ "caisi.04" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "statutory", "graph_id": "assessment#624", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 2, "b": [ "caisi.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "caisi.04" ] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "caisi.04" ] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "caisi.04" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "caisi.04" ] } }, "rationale_derived": "Unevidenced under this policy: no admissible signal sets a bound. Not counted: caisi.04 (no confirmed source (unaudited)).", "rationale_leads": "Capped at 2 by caisi.04 (R.4): Publishes little per model; findings mostly stay inside government.." }, "M": { "evaluator": "caisi", "dimension": "M", "value": 2, "anchor": 2, "signals": [ "caisi.07" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#625", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "caisi.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "caisi.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "caisi.07" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "caisi.07" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "caisi.07" ] } }, "rationale_derived": "Capped at 2 by caisi.07 (M.2): Methods partly published through NIST; per-evaluation methods not public..", "rationale_leads": "Capped at 2 by caisi.07 (M.2): Methods partly published through NIST; per-evaluation methods not public.." }, "X": { "evaluator": "caisi", "dimension": "X", "value": 3, "anchor": 3, "signals": [ "caisi.08", "caisi.16" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "resolution": { "rule": "X.6", "value": 3, "note": "The specific co-production rule (X.6) decides over the general legal-form floor: 3." }, "graph_id": "assessment#626", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "caisi.16" ], "h": false, "c": true, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "caisi.16" ], "h": false, "c": true, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "caisi.16" ], "h": false, "c": true, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "caisi.16" ], "h": false, "c": true, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "caisi.08", "caisi.16" ] } }, "rationale_derived": "Conflict: floor 4 from caisi.08 (X.7) against cap 3 from caisi.16 (X.6); resolved at 3 by X.6: The specific co-production rule (X.6) decides over the general legal-form floor: 3.", "rationale_leads": "Conflict: floor 4 from caisi.08 (X.7) against cap 3 from caisi.16 (X.6); resolved at 3 by X.6: The specific co-production rule (X.6) decides over the general legal-form floor: 3." } }, "signals": [ { "id": "caisi.01", "evaluator": "caisi", "dimension": "F", "direction": "for", "claim": "Public funding; formal access agreements with five developers.", "sources": [ "csa-caisi", "nist-caisi" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "F.9", "quote": "CAISI, which operates within NIST at the Department of Commerce", "quote_source": "csa-caisi", "graph_id": "signal#282", "source_graph_ids": [ "source#57", "source#135" ] }, { "id": "caisi.02", "evaluator": "caisi", "dimension": "A", "direction": "for", "claim": "More than 40 evaluations including unreleased models; classified CBRN and cyber work via the TRAINS taskforce.", "sources": [ "csa-caisi" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "A.4", "quote": "completed more than 40 evaluations — including assessments of frontier models not yet available to the public", "quote_source": "csa-caisi", "graph_id": "signal#283", "source_graph_ids": [ "source#57" ] }, { "id": "caisi.03", "evaluator": "caisi", "dimension": "G", "direction": "against", "claim": "Refocused in 2025 from broad safety research to demonstrable national-security risks; mandate is politically steerable.", "sources": [ "csa-caisi" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 3 }, "rule": "G.5", "quote": "repositioned the center as CAISI, shifting emphasis toward national security and cybersecurity risk reduction", "quote_source": "csa-caisi", "graph_id": "signal#284", "source_graph_ids": [ "source#57" ] }, { "id": "caisi.04", "evaluator": "caisi", "dimension": "R", "direction": "against", "claim": "Publishes little per model; findings mostly stay inside government.", "sources": [ "csa-caisi" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "R.4", "quote": "voluntary agreements, unclassified evaluations, no mandatory disclosure of findings", "quote_source": "csa-caisi", "graph_id": "signal#285", "source_graph_ids": [ "source#57" ] }, { "id": "caisi.05", "evaluator": "caisi", "dimension": "P", "direction": "against", "claim": "Authorized Scale AI, a company 49% owned by Meta, to run evaluations on its behalf.", "sources": [ "scale-framework", "longtermwiki-audit", "cnbc-scale-meta" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": null, "rule": "section 9", "bound_note": "Section 9 (mis-dimensioned fact): delegating evaluations to a lab-owned contractor is not a personnel fact; none of the P anchors (board seats, equity, advisory roles, hiring) describes it. Informational on P; re-recorded on X as new signal caisi.16 (X.6).", "quote": "Scale AI SEAL : Safety, Evaluation, and Alignment Lab; first third-party evaluator authorized by US AISI", "quote_source": "longtermwiki-audit", "graph_id": "signal#286", "source_graph_ids": [ "source#164", "source#104", "source#50" ] }, { "id": "caisi.06", "evaluator": "caisi", "dimension": "S", "direction": "for", "claim": "Government defines risk domains.", "sources": [ "scale-framework" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "S.4", "quote": "The government defines the risk domains it wants evaluated", "quote_source": "scale-framework", "graph_id": "signal#287", "source_graph_ids": [ "source#164" ] }, { "id": "caisi.07", "evaluator": "caisi", "dimension": "M", "direction": "against", "claim": "Methods partly published through NIST; per-evaluation methods not public.", "sources": [ "nist-caisi" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "M.2", "quote": "CAISI published a write-up on how AI models can cheat on agentic evaluations", "quote_source": "nist-caisi", "graph_id": "signal#288", "source_graph_ids": [ "source#135" ] }, { "id": "caisi.08", "evaluator": "caisi", "dimension": "X", "direction": "for", "claim": "No commercial products.", "sources": [ "nist-caisi", "csa-caisi-five-labs" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 4 }, "rule": "X.7", "quote": "lead unclassified evaluations of AI capabilities that may pose risks to national security", "quote_source": "nist-caisi", "graph_id": "signal#289", "source_graph_ids": [ "source#135", "source#56" ] }, { "id": "caisi.09", "evaluator": "caisi", "dimension": "G", "direction": "against", "claim": "Appropriation of about $10M in FY2026 within NIST's AI line; the director resigned in July 2026 and the NIST director is acting head.", "sources": [ "caisi-funding", "evaluators-ledger" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 3 }, "rule": "G.5", "quote": "$10 million from FY2026 appropriations", "quote_source": "caisi-funding", "graph_id": "signal#290", "source_graph_ids": [ "source#45", "source#81" ] }, { "id": "caisi.10", "evaluator": "caisi", "dimension": "F", "direction": "against", "claim": "Public appropriation of about $10M in FY2026; free pre-deployment access from labs is in-kind. No confirmed primary-filing negative on lab money is on file.", "sources": [ "caisi-funding", "nist-caisi" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 3 }, "rule": "F.9", "quote": "As of FY2026, CAISI has approximately $15 million in funding: $10 million from FY2026 appropriations", "quote_source": "caisi-funding", "graph_id": "signal#291", "source_graph_ids": [ "source#45", "source#135" ] }, { "id": "caisi.11", "evaluator": "caisi", "dimension": "G", "direction": "against", "claim": "The CAISI director resigned on 20 July 2026, three months after appointment — the third leadership change in six months.", "sources": [ "ifp-caisi-funding", "cnbc-caisi-fall" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2026-07-20", "quote": "resigned from his role as director just three months after he was picked for the job", "bound": { "cap": 3 }, "rule": "G.5", "quote_source": "cnbc-caisi-fall", "graph_id": "signal#292", "source_graph_ids": [ "source#98", "source#48" ] }, { "id": "caisi.12", "evaluator": "caisi", "dimension": "A", "direction": "for", "claim": "Five labs now give CAISI pre-deployment access (OpenAI and Anthropic since September 2025; Google DeepMind, Microsoft and xAI from 5 May 2026), including versions with guardrails stripped back; more than 40 evaluations completed, one of a foreign model without developer cooperation.", "sources": [ "csa-caisi-five-labs" ], "recorded": "2026-09-15", "curator": "yohei/claude v0-final", "as_of": "2026-05-05", "quote": "provide CAISI with model access that includes versions with safety guardrails stripped back", "bound": { "floor": 3 }, "rule": "A.4", "quote_source": "csa-caisi-five-labs", "graph_id": "signal#293", "source_graph_ids": [ "source#56" ] }, { "id": "caisi.13", "evaluator": "caisi", "dimension": "S", "direction": "against", "claim": "Participation is voluntary and agreements were jointly scoped with the two founding labs; the center can only assess models developers choose to share.", "sources": [ "csa-caisi-five-labs" ], "recorded": "2026-09-15", "curator": "yohei/claude v0-final", "as_of": "2026-05-05", "quote": "CAISI’s authority to conduct these evaluations rests entirely on voluntary participation", "bound": { "cap": 3 }, "rule": "S.4", "quote_source": "csa-caisi-five-labs", "graph_id": "signal#294", "source_graph_ids": [ "source#56" ] }, { "id": "caisi.14", "evaluator": "caisi", "dimension": "P", "direction": "for", "claim": "CAISI staff are NIST federal employees bound by the criminal conflict-of-interest statute (18 U.S.C. 208) and the post-employment statute (18 U.S.C. 207), which restricts former employees for one to two years after leaving.", "sources": [ "caisi-usc-208", "caisi-usc-207", "nist-caisi" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "floor": 3 }, "rule": "P.5", "quote": "within 1 year after the termination of his or her service or employment as such officer or employee", "quote_source": "caisi-usc-207", "note": "Statutory post-employment rules count as cooling-off (P.3); the conflict statute is a published mandatory recusal rule (P.5). Ties are disclosed on OGE forms, not publicly, so 3 not 4 (P.8). Replaces the unevidenced P value that would result from caisi.05 being informational.", "graph_id": "signal#295", "source_graph_ids": [ "source#47", "source#46", "source#135" ] }, { "id": "caisi.15", "evaluator": "caisi", "dimension": "G", "direction": "for", "claim": "CAISI is a NIST center whose staff are bound by the federal conflict-of-interest statute, 18 U.S.C. 208.", "sources": [ "caisi-usc-208", "nist-caisi" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "floor": 3 }, "rule": "G.2", "quote": "18 U.S. Code § 208 - Acts affecting a personal financial interest", "quote_source": "caisi-usc-208", "note": "Statutory civil-service conflict rules count as a conflict policy for a public body (G.2): floor 3. G.7 four needs an independent board and external review; none.", "graph_id": "signal#296", "source_graph_ids": [ "source#47", "source#135" ] }, { "id": "caisi.16", "evaluator": "caisi", "dimension": "X", "direction": "against", "claim": "CAISI partnered with Scale AI in February 2025 to jointly develop evaluation methods for frontier models, and authorized Scale SEAL as a third-party evaluator; Meta has held 49% of Scale AI since June 2025.", "sources": [ "scale-framework", "cnbc-scale-meta", "longtermwiki-audit" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "X.6", "quote": "Scale partnered with the U.S. Center for AI Standards and Innovation (CAISI), jointly developing new evaluation methods", "quote_source": "scale-framework", "note": "Co-producing evaluations with a vendor owned by a lab caps X at 3 (X.6); 49% exceeds the 20% control threshold used in F.1 and G.6. The partnership predates the Meta stake; whether it continued after June 2025 is a document request.", "graph_id": "signal#297", "source_graph_ids": [ "source#164", "source#50", "source#104" ] } ], "scores": { "lab": 72, "regulator": 72, "public": 73, "equal": 71 }, "scores_by_policy": { "leads": { "lab": { "score": 69, "band": "clear", "coverage": 8, "range": [ 69, 69 ] }, "regulator": { "score": 69, "band": "clear", "coverage": 8, "range": [ 69, 69 ] }, "public": { "score": 69, "band": "clear", "coverage": 8, "range": [ 69, 69 ] }, "equal": { "score": 69, "band": "clear", "coverage": 8, "range": [ 69, 69 ] } }, "standard": { "lab": { "score": 72, "band": "clear", "coverage": 7, "range": [ 62, 76 ] }, "regulator": { "score": 72, "band": "clear", "coverage": 7, "range": [ 61, 76 ] }, "public": { "score": 73, "band": "clear", "coverage": 7, "range": [ 59, 79 ] }, "equal": { "score": 71, "band": "clear", "coverage": 7, "range": [ 63, 75 ] } }, "against_interest": { "lab": { "score": 72, "band": "clear", "coverage": 7, "range": [ 62, 76 ] }, "regulator": { "score": 72, "band": "clear", "coverage": 7, "range": [ 61, 76 ] }, "public": { "score": 73, "band": "clear", "coverage": 7, "range": [ 59, 79 ] }, "equal": { "score": 71, "band": "clear", "coverage": 7, "range": [ 63, 75 ] } }, "spans": { "lab": { "score": 72, "band": "clear", "coverage": 7, "range": [ 62, 76 ] }, "regulator": { "score": 72, "band": "clear", "coverage": 7, "range": [ 61, 76 ] }, "public": { "score": 73, "band": "clear", "coverage": 7, "range": [ 59, 79 ] }, "equal": { "score": 71, "band": "clear", "coverage": 7, "range": [ 63, 75 ] } }, "primary": { "lab": { "score": 75, "band": "clear", "coverage": 2, "range": [ 14, 96 ] }, "regulator": { "score": 75, "band": "clear", "coverage": 2, "range": [ 15, 95 ] }, "public": { "score": 75, "band": "clear", "coverage": 2, "range": [ 23, 93 ] }, "equal": { "score": 75, "band": "clear", "coverage": 2, "range": [ 19, 94 ] } } }, "band": "clear", "coverage": 7, "floor": "M", "what_moves": { "lab": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 72 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 72 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 72 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 72 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 77, "band": "clear", "base": 72 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 75, "band": "clear", "base": 72 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 74, "band": "clear", "base": 72 } ], "regulator": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 76, "band": "clear", "base": 72 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 72 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 72 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 72 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 72 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 75, "band": "clear", "base": 72 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 72, "band": "clear", "base": 72 } ], "public": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 81, "band": "clear", "base": 73 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 73 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 73 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 73 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 77, "band": "clear", "base": 73 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 75, "band": "clear", "base": 73 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 73 } ], "equal": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 71 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 71 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 71 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 71 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 71 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 75, "band": "clear", "base": 71 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 71 } ] } }, { "id": "dreadnode", "name": "Dreadnode", "type": "vc", "hq": "Remote", "domains": [ "cyber" ], "confidence": "low", "summary": "Offensive-security firm named by Google DeepMind as an external expert on Gemini testing.", "what_would_move_the_score": "Published engagement terms and a first public report.", "role": "vendor", "dissent": { "lower": "Publication rights (1 to 0). Nothing from the Gemini engagement has reached the public: no summary, no finding, only the firm's name in Google's launch post and a spokesperson's sentence to Time. A name mention is not a lab-edited summary. Anchor 0 reads 'no publication, or lab approval required', and on the documented record there is no publication at all.", "higher": "Access depth (1 to 2). Google's external-testing programme shares pre-release models with selected experts for structured testing over a period before launch, and Dreadnode was named for both Gemini 2.5 Pro and Gemini 3. If the engagement ran for weeks against a pre-release checkpoint, that is extended-time access, anchor 2, rather than a one-off pre-release API test." }, "list_group": "commercial", "graph_id": "evaluator#208", "values": { "F": 2, "G": 1, "P": 2, "A": 1, "S": 2, "R": 2, "M": 2, "X": 2 }, "values_by_policy": { "leads": { "F": 2, "G": 1, "P": 2, "A": 1, "S": 2, "R": 2, "M": 2, "X": 2 }, "standard": { "F": 2, "G": 1, "P": null, "A": 1, "S": 2, "R": 2, "M": 2, "X": 2 }, "against_interest": { "F": 2, "G": 1, "P": null, "A": 1, "S": 2, "R": 2, "M": 1, "X": 2 }, "spans": { "F": 2, "G": 1, "P": null, "A": 1, "S": 2, "R": 2, "M": 2, "X": 2 }, "primary": { "F": null, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "dreadnode", "dimension": "F", "value": 2, "anchor": 2, "signals": [ "dreadnode.05" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#627", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "dreadnode.05" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "dreadnode.05" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "dreadnode.05" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "dreadnode.05" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "dreadnode.05" ] } }, "rationale_derived": "Capped at 2 by dreadnode.05 (F.3): Lab-paid engagements..", "rationale_leads": "Capped at 2 by dreadnode.05 (F.3): Lab-paid engagements.." }, "G": { "evaluator": "dreadnode", "dimension": "G", "value": 1, "anchor": 1, "signals": [ "dreadnode.02" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#628", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 1, "b": [ "dreadnode.02" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "dreadnode.02" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "dreadnode.02" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "dreadnode.02" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "dreadnode.02" ] } }, "rationale_derived": "Capped at 1 by dreadnode.02 (G.2): Venture-backed; no published COI policy found..", "rationale_leads": "Capped at 1 by dreadnode.02 (G.2): Venture-backed; no published COI policy found.." }, "P": { "evaluator": "dreadnode", "dimension": "P", "value": 2, "anchor": 2, "signals": [ "dreadnode.06" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#629", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "dreadnode.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "dreadnode.06" ] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "dreadnode.06" ] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "dreadnode.06" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "dreadnode.06" ] } }, "rationale_derived": "Unevidenced under this policy: no admissible signal sets a bound. Not counted: dreadnode.06 (no confirmed source (unverifiable)).", "rationale_leads": "Capped at 2 by dreadnode.06 (P.5): Undisclosed.." }, "A": { "evaluator": "dreadnode", "dimension": "A", "value": 1, "anchor": 1, "signals": [ "dreadnode.01" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "lab-controlled", "graph_id": "assessment#630", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 1, "b": [ "dreadnode.01" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "dreadnode.01" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "dreadnode.01" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "dreadnode.01" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "dreadnode.01" ] } }, "rationale_derived": "Floored at 1 by dreadnode.01 (A.1): Named by Google DeepMind as an external cyber expert on Gemini testing.. No admissible signal caps it.", "rationale_leads": "Floored at 1 by dreadnode.01 (A.1): Named by Google DeepMind as an external cyber expert on Gemini testing.. No admissible signal caps it." }, "S": { "evaluator": "dreadnode", "dimension": "S", "value": 2, "anchor": 2, "signals": [ "dreadnode.07" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#631", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "dreadnode.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "dreadnode.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "dreadnode.07" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "dreadnode.07" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "dreadnode.07" ] } }, "rationale_derived": "Capped at 2 by dreadnode.07 (S.2): Scope set by lab..", "rationale_leads": "Capped at 2 by dreadnode.07 (S.2): Scope set by lab.." }, "R": { "evaluator": "dreadnode", "dimension": "R", "value": 2, "anchor": 2, "signals": [ "dreadnode.03" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "lab-controlled", "graph_id": "assessment#632", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "dreadnode.03" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "dreadnode.03" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "dreadnode.03" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "dreadnode.03" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "dreadnode.03" ] } }, "rationale_derived": "Capped at 2 by dreadnode.03 (R.3): Evaluation terms and results are not public..", "rationale_leads": "Capped at 2 by dreadnode.03 (R.3): Evaluation terms and results are not public.." }, "M": { "evaluator": "dreadnode", "dimension": "M", "value": 2, "anchor": 2, "signals": [ "dreadnode.08", "dreadnode.09" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "resolution": { "rule": "M.2", "value": 2, "note": "See assessments.M.resolution: engagement-level closure (cap 1) against open self-directed benchmark code (floor 3); resolved at anchor 2, methods described in prose, consistent with Gray Swan and Scale." }, "evidence_limited": true, "graph_id": "assessment#633", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "dreadnode.08" ], "h": false, "c": true, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "dreadnode.08" ], "h": false, "c": true, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "dreadnode.08" ], "h": false, "c": false, "u": false, "x": [ "dreadnode.09" ] }, "spans": { "v": 2, "b": [ "dreadnode.08" ], "h": false, "c": true, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "dreadnode.08", "dreadnode.09" ] } }, "rationale_derived": "Conflict: floor 3 from dreadnode.09 (M.1) against cap 1 from dreadnode.08 (M.2); resolved at 2 by M.2: See assessments.M.resolution: engagement-level closure (cap 1) against open self-directed benchmark code (floor 3); resolved at anchor 2, methods described in prose, consistent with Gray Swan and Scale.", "rationale_leads": "Conflict: floor 3 from dreadnode.09 (M.1) against cap 1 from dreadnode.08 (M.2); resolved at 2 by M.2: See assessments.M.resolution: engagement-level closure (cap 1) against open self-directed benchmark code (floor 3); resolved at anchor 2, methods described in prose, consistent with Gray Swan and Scale." }, "X": { "evaluator": "dreadnode", "dimension": "X", "value": 2, "anchor": 2, "signals": [ "dreadnode.04" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#634", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "dreadnode.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "dreadnode.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "dreadnode.04" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "dreadnode.04" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "dreadnode.04" ] } }, "rationale_derived": "Capped at 2 by dreadnode.04 (X.1 (second sentence)): Sells offensive-security products..", "rationale_leads": "Capped at 2 by dreadnode.04 (X.1 (second sentence)): Sells offensive-security products.." } }, "signals": [ { "id": "dreadnode.01", "evaluator": "dreadnode", "dimension": "A", "direction": "for", "claim": "Named by Google DeepMind as an external cyber expert on Gemini testing.", "sources": [ "gdm-gemini-testing", "dreadnode-google-gemini3", "dreadnode-time-gdm-pledge" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 1 }, "rule": "A.1", "quote": "obtained independent assessments from industry experts like Apollo, Vaultis, Dreadnode and more", "quote_source": "dreadnode-google-gemini3", "graph_id": "signal#298", "source_graph_ids": [ "source#89", "source#62", "source#65" ] }, { "id": "dreadnode.02", "evaluator": "dreadnode", "dimension": "G", "direction": "against", "claim": "Venture-backed; no published COI policy found.", "sources": [ "gdm-gemini-testing", "dreadnode-seriesa", "dreadnode-securityweek" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 1 }, "rule": "G.2", "quote": "announced its $14 million Series A funding round, led by Decibel", "quote_source": "dreadnode-seriesa", "graph_id": "signal#299", "source_graph_ids": [ "source#89", "source#64", "source#63" ] }, { "id": "dreadnode.03", "evaluator": "dreadnode", "dimension": "R", "direction": "against", "claim": "Evaluation terms and results are not public.", "sources": [ "gdm-gemini-testing", "dreadnode-time-gdm-pledge", "dreadnode-arxiv-airtbench" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "R.3", "quote": "a “diverse group of external experts,” including Apollo Research, Dreadnode, and Vaultis", "quote_source": "dreadnode-time-gdm-pledge", "graph_id": "signal#300", "source_graph_ids": [ "source#89", "source#65", "source#60" ] }, { "id": "dreadnode.04", "evaluator": "dreadnode", "dimension": "X", "direction": "against", "claim": "Sells offensive-security products.", "sources": [ "gdm-gemini-testing", "dreadnode-seriesa", "dreadnode-securityweek" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "X.1 (second sentence)", "quote": "two new products — Strikes and Spyglass — that form the core of its platform", "quote_source": "dreadnode-securityweek", "graph_id": "signal#301", "source_graph_ids": [ "source#89", "source#64", "source#63" ] }, { "id": "dreadnode.05", "evaluator": "dreadnode", "dimension": "F", "direction": "against", "claim": "Lab-paid engagements.", "sources": [ "gdm-gemini-testing", "dreadnode-google-gemini3" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "F.3", "quote": "obtained independent assessments from industry experts like Apollo, Vaultis, Dreadnode and more", "quote_source": "dreadnode-google-gemini3", "graph_id": "signal#302", "source_graph_ids": [ "source#89", "source#62" ] }, { "id": "dreadnode.06", "evaluator": "dreadnode", "dimension": "P", "direction": "against", "claim": "Undisclosed.", "sources": [ "gdm-gemini-testing" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "P.5", "graph_id": "signal#303", "source_graph_ids": [ "source#89" ] }, { "id": "dreadnode.07", "evaluator": "dreadnode", "dimension": "S", "direction": "against", "claim": "Scope set by lab.", "sources": [ "gdm-gemini-testing", "dreadnode-time-gdm-pledge" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "S.2", "quote": "a “diverse group of external experts,” including Apollo Research, Dreadnode, and Vaultis", "quote_source": "dreadnode-time-gdm-pledge", "graph_id": "signal#304", "source_graph_ids": [ "source#89", "source#65" ] }, { "id": "dreadnode.08", "evaluator": "dreadnode", "dimension": "M", "direction": "against", "claim": "Methods and results are closed; no public methodology.", "sources": [ "gdm-gemini-testing", "dreadnode-google-gemini3" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 1 }, "rule": "M.2", "quote": "obtained independent assessments from industry experts like Apollo, Vaultis, Dreadnode and more", "quote_source": "dreadnode-google-gemini3", "graph_id": "signal#305", "source_graph_ids": [ "source#89", "source#62" ] }, { "id": "dreadnode.09", "evaluator": "dreadnode", "dimension": "M", "direction": "for", "claim": "Publishes AIRTBench, an AI red-teaming benchmark with open code (Apache-2.0) and results on Claude 3.7 Sonnet, Gemini 2.5 Pro and GPT-4.5; the Gemini engagement itself remains closed.", "sources": [ "dreadnode-arxiv-airtbench", "dreadnode-github-airtbench" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "floor": 3 }, "rule": "M.1", "quote": "challenges from the Crucible challenge environment on the Dreadnode platform", "quote_source": "dreadnode-arxiv-airtbench", "note": "Needed because dreadnode.08's claim ('no public methodology') is contradicted at the organization level; M.1 gives 3 for open code alone. Rule 9 does not hold this floor down: AIRTBench is a published evaluation of frontier models. Conflicts with the cap of 1 on dreadnode.08; see the M resolution.", "graph_id": "signal#306", "source_graph_ids": [ "source#60", "source#61" ] } ], "scores": { "lab": 42, "regulator": 42, "public": 44, "equal": 43 }, "scores_by_policy": { "leads": { "lab": { "score": 43, "band": "conditional", "coverage": 8, "range": [ 43, 43 ] }, "regulator": { "score": 43, "band": "conditional", "coverage": 8, "range": [ 43, 43 ] }, "public": { "score": 45, "band": "conditional", "coverage": 8, "range": [ 45, 45 ] }, "equal": { "score": 44, "band": "conditional", "coverage": 8, "range": [ 44, 44 ] } }, "standard": { "lab": { "score": 42, "band": "conditional", "coverage": 7, "range": [ 38, 48 ] }, "regulator": { "score": 42, "band": "conditional", "coverage": 7, "range": [ 38, 48 ] }, "public": { "score": 44, "band": "conditional", "coverage": 7, "range": [ 38, 53 ] }, "equal": { "score": 43, "band": "conditional", "coverage": 7, "range": [ 38, 50 ] } }, "against_interest": { "lab": { "score": 40, "band": "conditional", "coverage": 7, "range": [ 36, 46 ] }, "regulator": { "score": 39, "band": "conditional", "coverage": 7, "range": [ 35, 45 ] }, "public": { "score": 43, "band": "conditional", "coverage": 7, "range": [ 36, 51 ] }, "equal": { "score": 39, "band": "conditional", "coverage": 7, "range": [ 34, 47 ] } }, "spans": { "lab": { "score": 42, "band": "conditional", "coverage": 7, "range": [ 38, 48 ] }, "regulator": { "score": 42, "band": "conditional", "coverage": 7, "range": [ 38, 48 ] }, "public": { "score": 44, "band": "conditional", "coverage": 7, "range": [ 38, 53 ] }, "equal": { "score": 43, "band": "conditional", "coverage": 7, "range": [ 38, 50 ] } }, "primary": { "lab": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "regulator": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "public": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "equal": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] } } }, "band": "conditional", "coverage": 7, "floor": "G", "what_moves": { "lab": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 47, "band": "conditional", "base": 42 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 44, "band": "conditional", "base": 42 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 48, "band": "conditional", "base": 42 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 47, "band": "conditional", "base": 42 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 46, "band": "conditional", "base": 42 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 45, "band": "conditional", "base": 42 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 44, "band": "conditional", "base": 42 } ], "regulator": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 46, "band": "conditional", "base": 42 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 44, "band": "conditional", "base": 42 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 47, "band": "conditional", "base": 42 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 47, "band": "conditional", "base": 42 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 46, "band": "conditional", "base": 42 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 44, "band": "conditional", "base": 42 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 42, "band": "conditional", "base": 42 } ], "public": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 51, "band": "conditional", "base": 44 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 49, "band": "conditional", "base": 44 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 46, "band": "conditional", "base": 44 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 47, "band": "conditional", "base": 44 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 50, "band": "conditional", "base": 44 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 46, "band": "conditional", "base": 44 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 46, "band": "conditional", "base": 44 } ], "equal": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 46, "band": "conditional", "base": 43 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 46, "band": "conditional", "base": 43 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 46, "band": "conditional", "base": 43 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 46, "band": "conditional", "base": 43 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 46, "band": "conditional", "base": 43 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 46, "band": "conditional", "base": 43 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 46, "band": "conditional", "base": 43 } ] } }, { "id": "epoch", "name": "Epoch AI", "type": "nonprofit", "hq": "Remote", "domains": [ "benchmarks" ], "confidence": "high", "summary": "Capability-tracking and benchmark nonprofit; the FrontierMath episode is the canonical undisclosed-funding case.", "what_would_move_the_score": "A standing rule against confidential lab funding of any evaluation asset.", "role": "referee", "dissent": { "lower": "Funding should be 1. OpenAI commissioned and owns FrontierMath, the asset Epoch's reputation rests on, and held approval over disclosing that fact for a year. OpenAI, Google DeepMind, xAI and Anthropic are all paying clients, the revenue split is undisclosed, and Epoch invests in AI stocks. 'Every major lab' as clients with an undisclosed share reads as F.5's material lab revenue, not F.3's occasional fee.", "higher": "Funding should be 3. Coefficient's $25.1M over eight grants dwarfs client fees, and Epoch discloses every donation above $70,000 and every client by name and year. It charges at least consultant rates so labs do not get subsidised work, keeps a 50-problem holdout OpenAI cannot see, and publishes evaluations of every lab's models on the same benchmark, including results OpenAI did not commission." }, "list_group": "referee", "graph_id": "evaluator#209", "values": { "F": 2, "G": 2, "P": 2, "A": 1, "S": 3, "R": 2, "M": 3, "X": 2 }, "values_by_policy": { "leads": { "F": 2, "G": 2, "P": 2, "A": 1, "S": 3, "R": 2, "M": 3, "X": 2 }, "standard": { "F": 2, "G": 2, "P": 2, "A": 1, "S": 3, "R": 2, "M": 3, "X": 2 }, "against_interest": { "F": 2, "G": 2, "P": 2, "A": 1, "S": 3, "R": 2, "M": 3, "X": 2 }, "spans": { "F": 2, "G": 2, "P": null, "A": 1, "S": 3, "R": 2, "M": null, "X": 2 }, "primary": { "F": null, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "anchor": 2, "assessed": "2026-09-15", "assessor": "yohei/claude v0-final", "dimension": "F", "evaluator": "epoch", "rationale": "Anchor 2 on funding given the cited signals. Curator disclosure (2026-09-15): the curator holds small public-market shareholdings in Google and Meta. This evaluator has a confirmed ledger tie to Google (contract row T35). Per the disclosure Rule, this assessment is flagged for public review: the rationale names the holding so readers can weigh it, and any reader can file a correction through the contribution path.", "signals": [ "epoch.01", "epoch.05", "epoch.11", "epoch.13", "epoch.15" ], "value": 2, "resolution": { "rule": "F.3", "value": 2, "note": "F.3 fee cap decides; the philanthropic floor is itself limited to 3 by F.6." }, "graph_id": "assessment#635", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 2, "b": [ "epoch.01", "epoch.11", "epoch.13" ], "h": false, "c": true, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "epoch.01", "epoch.11", "epoch.13" ], "h": false, "c": true, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "epoch.01", "epoch.11", "epoch.13" ], "h": false, "c": true, "u": false, "x": [ "epoch.15" ] }, "spans": { "v": 2, "b": [ "epoch.01", "epoch.11", "epoch.13" ], "h": false, "c": true, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "epoch.01", "epoch.05", "epoch.11", "epoch.13", "epoch.15" ] } }, "rationale_derived": "Conflict: floor 3 from epoch.05 (F.6) against cap 2 from epoch.01, epoch.11, epoch.13 (F.3, F.3, F.3); resolved at 2 by F.3: F.3 fee cap decides; the philanthropic floor is itself limited to 3 by F.6.", "rationale_leads": "Conflict: floor 3 from epoch.05 (F.6) against cap 2 from epoch.01, epoch.11, epoch.13 (F.3, F.3, F.3); resolved at 2 by F.3: F.3 fee cap decides; the philanthropic floor is itself limited to 3 by F.6." }, "G": { "anchor": 2, "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "dimension": "G", "evaluator": "epoch", "signals": [ "epoch.06" ], "value": 2, "graph_id": "assessment#636", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 2, "b": [ "epoch.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "epoch.06" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "epoch.06" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "epoch.06" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "epoch.06" ] } }, "rationale_derived": "Floored at 2 by epoch.06 (G.1): Nonprofit.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by epoch.06 (G.1): Nonprofit.. No admissible signal caps it." }, "P": { "anchor": 2, "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "dimension": "P", "evaluator": "epoch", "signals": [ "epoch.09" ], "value": 2, "graph_id": "assessment#637", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 2, "b": [ "epoch.09" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "epoch.09" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "epoch.09" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "epoch.09" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "epoch.09" ] } }, "rationale_derived": "Floored at 2 by epoch.09 (P.6): No lab roles found.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by epoch.09 (P.6): No lab roles found.. No admissible signal caps it." }, "A": { "anchor": 1, "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "dimension": "A", "evaluator": "epoch", "signals": [ "epoch.07", "epoch.18" ], "value": 1, "mechanism": "lab-controlled", "graph_id": "assessment#638", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 1, "b": [ "epoch.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "epoch.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "epoch.07" ], "h": false, "c": false, "u": false, "x": [ "epoch.18" ] }, "spans": { "v": 1, "b": [ "epoch.07" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "epoch.07", "epoch.18" ] } }, "rationale_derived": "Capped at 1 by epoch.07 (A.1): Cannot share the commissioned set with other labs without OpenAI's permission.. Floors up to 1 from epoch.18 do not exceed the cap.", "rationale_leads": "Capped at 1 by epoch.07 (A.1): Cannot share the commissioned set with other labs without OpenAI's permission.. Floors up to 1 from epoch.18 do not exceed the cap." }, "S": { "anchor": 3, "assessed": "2026-09-14", "assessor": "yohei/claude v0", "dimension": "S", "evaluator": "epoch", "signals": [ "epoch.08", "epoch.17" ], "value": 3, "graph_id": "assessment#639", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "epoch.17" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "epoch.17" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "epoch.17" ], "h": false, "c": false, "u": false, "x": [ "epoch.08" ] }, "spans": { "v": 3, "b": [ "epoch.17" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "epoch.08", "epoch.17" ] } }, "rationale_derived": "Capped at 3 by epoch.17 (S.6): FrontierMath, the asset on which Epoch evaluates OpenAI and other labs, was commissioned and is owned by OpenAI, which holds the problems and solutions outside the 50-problem holdout.. Floors up to 3 from epoch.08 do not exceed the cap.", "rationale_leads": "Capped at 3 by epoch.17 (S.6): FrontierMath, the asset on which Epoch evaluates OpenAI and other labs, was commissioned and is owned by OpenAI, which holds the problems and solutions outside the 50-problem holdout.. Floors up to 3 from epoch.08 do not exceed the cap." }, "R": { "anchor": 2, "assessed": "2026-09-14", "assessor": "yohei/claude v0", "dimension": "R", "evaluator": "epoch", "signals": [ "epoch.02", "epoch.03" ], "value": 2, "mechanism": "lab-controlled", "graph_id": "assessment#640", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "epoch.02" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "epoch.02" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "epoch.02" ], "h": false, "c": false, "u": false, "x": [ "epoch.03" ] }, "spans": { "v": 2, "b": [ "epoch.02" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "epoch.02", "epoch.03" ] } }, "rationale_derived": "Capped at 2 by epoch.02 (R.2): Contributors were not told the funder; disclosure came after the fact.. Floors up to 2 from epoch.03 do not exceed the cap.", "rationale_leads": "Capped at 2 by epoch.02 (R.2): Contributors were not told the funder; disclosure came after the fact.. Floors up to 2 from epoch.03 do not exceed the cap." }, "M": { "anchor": 3, "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "dimension": "M", "evaluator": "epoch", "signals": [ "epoch.04" ], "value": 3, "graph_id": "assessment#641", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 3, "b": [ "epoch.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "epoch.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "epoch.04" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "epoch.04" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "epoch.04" ] } }, "rationale_derived": "Floored at 3 by epoch.04 (M.1): Rigorous public data on compute, training runs and capability trends.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by epoch.04 (M.1): Rigorous public data on compute, training runs and capability trends.. No admissible signal caps it." }, "X": { "anchor": 2, "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "dimension": "X", "evaluator": "epoch", "signals": [ "epoch.10", "epoch.14", "epoch.16" ], "value": 2, "open_questions": [ "Which AI stocks Epoch holds and whether they include developers it evaluates." ], "resolution": { "rule": "X.3", "value": 2, "note": "epoch.10's raw floor of 4 is held to 3 by C15/C16 and still exceeds the X.3 cap of 2; X.3 decides because the client list is on Epoch's own page." }, "graph_id": "assessment#642", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "epoch.16" ], "h": true, "c": true, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "epoch.16" ], "h": true, "c": true, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "epoch.16" ], "h": true, "c": true, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "epoch.16" ], "h": false, "c": false, "u": false, "x": [ "epoch.10" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "epoch.10", "epoch.14", "epoch.16" ] } }, "rationale_derived": "Conflict: floor 3 from epoch.10 (X.7) against cap 2 from epoch.16 (X.3); resolved at 2 by X.3: epoch.10's raw floor of 4 is held to 3 by C15/C16 and still exceeds the X.3 cap of 2; X.3 decides because the client list is on Epoch's own page.", "rationale_leads": "Conflict: floor 3 from epoch.10 (X.7) against cap 2 from epoch.16 (X.3); resolved at 2 by X.3: epoch.10's raw floor of 4 is held to 3 by C15/C16 and still exceeds the X.3 cap of 2; X.3 decides because the client list is on Epoch's own page." } }, "signals": [ { "id": "epoch.01", "evaluator": "epoch", "dimension": "F", "direction": "against", "claim": "OpenAI commissioned and owns most of FrontierMath and had the problems and solutions; a contract barred disclosing this until the o3 launch.", "sources": [ "epoch-clarify", "decoder-epoch" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "quote": "we cannot share the questions and answers with other parties without written permission from OpenAI", "bound": { "cap": 2 }, "rule": "F.3", "quote_source": "epoch-clarify", "graph_id": "signal#307", "source_graph_ids": [ "source#66", "source#59" ] }, { "id": "epoch.02", "evaluator": "epoch", "dimension": "R", "direction": "against", "claim": "Contributors were not told the funder; disclosure came after the fact.", "sources": [ "techcrunch-epoch", "epoch-clarify" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "R.2", "quote": "Per our agreement, we needed OpenAI’s permission before publicly disclosing their involvement", "quote_source": "epoch-clarify", "graph_id": "signal#308", "source_graph_ids": [ "source#177", "source#66" ] }, { "id": "epoch.03", "evaluator": "epoch", "dimension": "R", "direction": "for", "claim": "After the disclosure failure, created a 50-problem holdout OpenAI cannot see.", "sources": [ "epoch-clarify" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "R.1", "quote": "We are finalizing a 50-problem set for which OpenAI will only receive the problem statements and not the solutions", "quote_source": "epoch-clarify", "graph_id": "signal#309", "source_graph_ids": [ "source#66" ] }, { "id": "epoch.04", "evaluator": "epoch", "dimension": "M", "direction": "for", "claim": "Rigorous public data on compute, training runs and capability trends.", "sources": [ "techcrunch-epoch" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "M.1", "graph_id": "signal#310", "source_graph_ids": [ "source#177" ] }, { "id": "epoch.05", "evaluator": "epoch", "dimension": "F", "direction": "for", "claim": "Primarily philanthropically funded (Open Philanthropy).", "sources": [ "techcrunch-epoch" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "F.6", "quote": "Epoch AI, a nonprofit primarily funded by Open Philanthropy", "quote_source": "techcrunch-epoch", "graph_id": "signal#311", "source_graph_ids": [ "source#177" ] }, { "id": "epoch.06", "evaluator": "epoch", "dimension": "G", "direction": "for", "claim": "Nonprofit.", "sources": [ "techcrunch-epoch" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "G.1", "quote": "Epoch AI, a nonprofit primarily funded by Open Philanthropy", "quote_source": "techcrunch-epoch", "graph_id": "signal#312", "source_graph_ids": [ "source#177" ] }, { "id": "epoch.07", "evaluator": "epoch", "dimension": "A", "direction": "against", "claim": "Cannot share the commissioned set with other labs without OpenAI's permission.", "sources": [ "epoch-clarify" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 1 }, "rule": "A.1", "quote": "we cannot share the questions and answers with other parties without written permission from OpenAI", "quote_source": "epoch-clarify", "graph_id": "signal#313", "source_graph_ids": [ "source#66" ] }, { "id": "epoch.08", "evaluator": "epoch", "dimension": "S", "direction": "for", "claim": "Evaluates any model on FrontierMath at its discretion.", "sources": [ "epoch-clarify" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "S.5", "quote": "Epoch AI is free to conduct and publish evaluations of any models using the FrontierMath problem set", "quote_source": "epoch-clarify", "graph_id": "signal#314", "source_graph_ids": [ "source#66" ] }, { "id": "epoch.09", "evaluator": "epoch", "dimension": "P", "direction": "for", "claim": "No lab roles found.", "sources": [ "techcrunch-epoch" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "P.6", "graph_id": "signal#315", "source_graph_ids": [ "source#177" ] }, { "id": "epoch.10", "evaluator": "epoch", "dimension": "X", "direction": "for", "claim": "No commercial products.", "sources": [ "techcrunch-epoch" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 4 }, "rule": "X.7", "graph_id": "signal#316", "source_graph_ids": [ "source#177" ] }, { "id": "epoch.11", "evaluator": "epoch", "dimension": "F", "direction": "against", "claim": "Transparency page lists Google DeepMind and OpenAI as paying consultation clients and a $600K Tallinn DAF gift; Coefficient grants of about $24.5M per index.", "sources": [ "epoch-transparency", "coefficient-index" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-11", "bound": { "cap": 2 }, "rule": "F.3", "quote": "Google DeepMind 2026 Evaluating the AI co-mathematician on FrontierMath 2025 Model Evaluations", "quote_source": "epoch-transparency", "graph_id": "signal#317", "source_graph_ids": [ "source#70", "source#52" ] }, { "id": "epoch.12", "evaluator": "epoch", "dimension": "M", "direction": "for", "claim": "Publishes a transparency page listing donations above $70K and paying clients.", "sources": [ "epoch-transparency" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": null, "rule": "M.2", "bound_note": "Informational: a funding-transparency page is not evidence about method transparency; no M anchor or rule maps it. Recommend re-dimensioning to F (for) where the disclosure counts toward anchor 4's 'disclosed'.", "quote": "Here, we list only donations of $70,000 USD or more", "quote_source": "epoch-transparency", "graph_id": "signal#318", "source_graph_ids": [ "source#70" ] }, { "id": "epoch.13", "evaluator": "epoch", "dimension": "F", "direction": "against", "claim": "Transparency page lists eight Coefficient grants (largest $8,500,000 in April 2025; about $25M cumulative), a $600,000 Tallinn DAF gift, $195,000 from SFF, and paying clients including OpenAI, Google DeepMind, xAI, an Anthropic pilot, the EU AI Office, Sequoia Capital Global Equities and Bridgewater.", "sources": [ "epoch-transparency-page" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2026", "bound": { "cap": 2 }, "rule": "F.3", "quote": "xAI 2025 Model Evaluations Anthropic 2024 Small pilot for new benchmark", "quote_source": "epoch-transparency-page", "graph_id": "signal#319", "source_graph_ids": [ "source#69" ] }, { "id": "epoch.14", "evaluator": "epoch", "dimension": "X", "direction": "against", "claim": "Epoch states it invests part of its funds in semiconductor and AI stocks as part of a diversified portfolio.", "sources": [ "epoch-transparency-page" ], "recorded": "2026-09-15", "curator": "yohei/claude v0-final", "as_of": "2026", "quote": "We invest part of our funds in semiconductor and AI stocks as part of a diversified portfolio", "bound": { "cap": 3 }, "rule": "X.5", "quote_source": "epoch-transparency-page", "graph_id": "signal#320", "source_graph_ids": [ "source#69" ] }, { "id": "epoch.15", "evaluator": "epoch", "dimension": "F", "direction": "for", "claim": "Epoch says it charges lab and industry clients at least industry-consultant rates so it is not subsidising their work, and commits to disclosing sponsorship and data-access agreements; its eight Coefficient grants sum to $25.1M by its own list.", "sources": [ "epoch-transparency-page" ], "recorded": "2026-09-15", "curator": "yohei/claude v0-final", "as_of": "2026", "quote": "We aim to charge prices at least on par with industry consultants", "bound": { "floor": 2 }, "rule": "F.3", "quote_source": "epoch-transparency-page", "graph_id": "signal#321", "source_graph_ids": [ "source#69" ] }, { "id": "epoch.16", "evaluator": "epoch", "dimension": "X", "direction": "against", "claim": "Epoch's transparency page lists paid consultation and evaluation work for OpenAI (2024 to 2026), Google DeepMind (2024 to 2026), xAI (2025) and an Anthropic pilot (2024).", "sources": [ "epoch-transparency-page" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 2 }, "rule": "X.3", "quote": "OpenAI 2026 FrontierMath Data Audit", "quote_source": "epoch-transparency-page", "note": "Paid consultation clients listed on a transparency page count (X.3). Also on the page: 'xAI 2025 Model Evaluations Anthropic 2024 Small pilot for new benchmark'. Derived X is 2, not the stored 3.", "graph_id": "signal#322", "source_graph_ids": [ "source#69" ] }, { "id": "epoch.17", "evaluator": "epoch", "dimension": "S", "direction": "against", "claim": "FrontierMath, the asset on which Epoch evaluates OpenAI and other labs, was commissioned and is owned by OpenAI, which holds the problems and solutions outside the 50-problem holdout.", "sources": [ "epoch-clarify" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "S.6", "quote": "OpenAI commissioned Epoch AI to produce 300 advanced math problems for AI evaluation", "quote_source": "epoch-clarify", "note": "A lab-commissioned evaluation asset used to evaluate that lab is co-production in the sense of S.6 (cap 3). Not binding today (floor is 3) but rule-required.", "graph_id": "signal#323", "source_graph_ids": [ "source#66" ] }, { "id": "epoch.18", "evaluator": "epoch", "dimension": "A", "direction": "for", "claim": "Epoch reports pre-release access to evaluate GPT-5.4 (March 2026) and Meta's Muse Spark on FrontierMath.", "sources": [ "epoch-gpt54-frontiermath", "epoch-musespark-x" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "floor": 1 }, "rule": "A.1", "quote": "We had pre-release access to evaluate the model", "quote_source": "epoch-gpt54-frontiermath", "note": "Deepest documented engagement is pre-release API access; no safeguards-off, extended-window, or weights-level access is documented, so anchor 1 (A.1).", "graph_id": "signal#324", "source_graph_ids": [ "source#67", "source#68" ] } ], "scores": { "lab": 51, "regulator": 53, "public": 53, "equal": 53 }, "scores_by_policy": { "leads": { "lab": { "score": 51, "band": "conditional", "coverage": 8, "range": [ 51, 51 ] }, "regulator": { "score": 53, "band": "conditional", "coverage": 8, "range": [ 53, 53 ] }, "public": { "score": 53, "band": "conditional", "coverage": 8, "range": [ 53, 53 ] }, "equal": { "score": 53, "band": "conditional", "coverage": 8, "range": [ 53, 53 ] } }, "standard": { "lab": { "score": 51, "band": "conditional", "coverage": 8, "range": [ 51, 51 ] }, "regulator": { "score": 53, "band": "conditional", "coverage": 8, "range": [ 53, 53 ] }, "public": { "score": 53, "band": "conditional", "coverage": 8, "range": [ 53, 53 ] }, "equal": { "score": 53, "band": "conditional", "coverage": 8, "range": [ 53, 53 ] } }, "against_interest": { "lab": { "score": 51, "band": "conditional", "coverage": 8, "range": [ 51, 51 ] }, "regulator": { "score": 53, "band": "conditional", "coverage": 8, "range": [ 53, 53 ] }, "public": { "score": 53, "band": "conditional", "coverage": 8, "range": [ 53, 53 ] }, "equal": { "score": 53, "band": "conditional", "coverage": 8, "range": [ 53, 53 ] } }, "spans": { "lab": { "score": 49, "band": "conditional", "coverage": 6, "range": [ 40, 59 ] }, "regulator": { "score": 50, "band": "conditional", "coverage": 6, "range": [ 40, 60 ] }, "public": { "score": 52, "band": "conditional", "coverage": 6, "range": [ 41, 61 ] }, "equal": { "score": 50, "band": "conditional", "coverage": 6, "range": [ 38, 63 ] } }, "primary": { "lab": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "regulator": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "public": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "equal": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] } } }, "band": "conditional", "coverage": 8, "floor": "A", "what_moves": { "lab": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 51 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 53, "band": "conditional", "base": 51 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 54, "band": "conditional", "base": 51 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 56, "band": "clear", "base": 51 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 55, "band": "conditional", "base": 51 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 55, "band": "conditional", "base": 51 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 54, "band": "conditional", "base": 51 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 53, "band": "conditional", "base": 51 } ], "regulator": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 55, "band": "conditional", "base": 53 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 55, "band": "conditional", "base": 53 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 57, "band": "clear", "base": 53 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 57, "band": "conditional", "base": 53 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 55, "band": "conditional", "base": 53 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 53, "band": "conditional", "base": 53 } ], "public": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 53 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 54, "band": "clear", "base": 53 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 55, "band": "conditional", "base": 53 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 57, "band": "conditional", "base": 53 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 54, "band": "conditional", "base": 53 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 54, "band": "conditional", "base": 53 } ], "equal": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 56, "band": "clear", "base": 53 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 } ] } }, { "id": "equistamp", "name": "EquiStamp", "type": "private", "hq": "Remote", "domains": [ "assurance", "autonomy" ], "confidence": "low", "summary": "Private evaluation-services company; leads two EU AI Office technical-assistance lots (loss of control with METR and Epoch; harmful manipulation with Transluce). Clients named are evaluators and governments.", "what_would_move_the_score": "Funding and ownership disclosure; a lab-money policy.", "role": "referee", "dissent": { "lower": "Governance (G) at 1 could be 0. The public record offers no case: nothing shows a lab or lab investor holding any stake in EquiStamp, and the company says it is converting to a public benefit corporation. The only lower reading is evidentiary, not substantive: with no funding or ownership disclosure at all, a 20% lab stake cannot be excluded, and the ledger relies on the company's own site.", "higher": "Governance (G) at 1 could be 2. EquiStamp states a public benefit commitment to advance AI safety research over profit and is converting to a PBC, whose charter would bind directors to that purpose; its named clients are evaluators and governments, not labs. Anchor 2 asks for a published conflict policy; a PBC charter with a stated public benefit is the nearest published instrument and the company is not venture-backed on any record." }, "list_group": "referee", "graph_id": "evaluator#210", "values": { "F": 3, "G": 1, "P": 2, "A": 1, "S": 2, "R": 2, "M": 2, "X": 3 }, "values_by_policy": { "leads": { "F": 3, "G": 1, "P": 2, "A": 1, "S": 2, "R": 2, "M": 2, "X": 3 }, "standard": { "F": 3, "G": 1, "P": null, "A": 1, "S": 2, "R": 2, "M": null, "X": 3 }, "against_interest": { "F": 3, "G": 1, "P": null, "A": 1, "S": 2, "R": 2, "M": null, "X": null }, "spans": { "F": 3, "G": null, "P": null, "A": 1, "S": 2, "R": null, "M": null, "X": 3 }, "primary": { "F": 3, "G": null, "P": null, "A": null, "S": 2, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "equistamp", "dimension": "F", "value": 3, "anchor": 3, "signals": [ "equistamp.01", "equistamp.02" ], "rationale": "Anchor 3 from the cited rows; low confidence, imported evidence.", "assessed": "2026-09-14", "assessor": "yohei/claude v0.4", "graph_id": "assessment#643", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 3, "b": [ "equistamp.02" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "equistamp.02" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "equistamp.02" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "equistamp.02" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": 3, "b": [ "equistamp.01" ], "h": false, "c": false, "u": false, "x": [ "equistamp.02" ] } }, "rationale_derived": "Capped at 3 by equistamp.02 (F.10): Private company; no funding or ownership disclosed.. Floors up to 3 from equistamp.01 do not exceed the cap.", "rationale_leads": "Capped at 3 by equistamp.02 (F.10): Private company; no funding or ownership disclosed.. Floors up to 3 from equistamp.01 do not exceed the cap." }, "G": { "evaluator": "equistamp", "dimension": "G", "value": 1, "anchor": 1, "signals": [ "equistamp.03" ], "rationale": "Anchor 1 from the cited rows; low confidence, imported evidence. Evidence-limited: the cited sources are imported or unaudited only, so the value is held at 2 (the reading on the record would be 1) until a live source is confirmed.", "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#644", "evidence_tier": "tier 2 (ledger)", "derived": { "leads": { "v": 1, "b": [ "equistamp.03" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "equistamp.03" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "equistamp.03" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "equistamp.03" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "equistamp.03" ] } }, "rationale_derived": "Capped at 1 by equistamp.03 (G.2): No COI policy on the site..", "rationale_leads": "Capped at 1 by equistamp.03 (G.2): No COI policy on the site.." }, "P": { "evaluator": "equistamp", "dimension": "P", "value": 2, "anchor": 2, "signals": [ "equistamp.04" ], "rationale": "Anchor 2 from the cited rows; low confidence, imported evidence.", "assessed": "2026-09-14", "assessor": "yohei/claude v0.4", "graph_id": "assessment#645", "evidence_tier": "tier 2 (ledger)", "derived": { "leads": { "v": 2, "b": [ "equistamp.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "equistamp.04" ] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "equistamp.04" ] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "equistamp.04" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "equistamp.04" ] } }, "rationale_derived": "Unevidenced under this policy: no admissible signal sets a bound. Not counted: equistamp.04 (no confirmed source (imported)).", "rationale_leads": "Capped at 2 by equistamp.04 (P.6): Founders' prior affiliations and any lab ties not established.." }, "A": { "evaluator": "equistamp", "dimension": "A", "value": 1, "anchor": 1, "signals": [ "equistamp.05" ], "rationale": "Anchor 2 from the cited rows; low confidence, imported evidence.", "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "unknown", "evidence_limited": true, "graph_id": "assessment#646", "evidence_tier": "tier 2 (ledger)", "derived": { "leads": { "v": 1, "b": [ "equistamp.05" ], "h": true, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "equistamp.05" ], "h": true, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "equistamp.05" ], "h": true, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "equistamp.05" ], "h": true, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "equistamp.05" ] } }, "rationale_derived": "Capped at 1 by equistamp.05 (A.6): Access is through client engagements; no pre-release lab access documented.. Held: C14: sources not all confirmed.", "rationale_leads": "Capped at 1 by equistamp.05 (A.6): Access is through client engagements; no pre-release lab access documented.. Held: C14: sources not all confirmed." }, "S": { "evaluator": "equistamp", "dimension": "S", "value": 2, "anchor": 2, "signals": [ "equistamp.06" ], "rationale": "Anchor 3 from the cited rows; low confidence, imported evidence.", "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#647", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 2, "b": [ "equistamp.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "equistamp.06" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "equistamp.06" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "equistamp.06" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": 2, "b": [ "equistamp.06" ], "h": false, "c": false, "u": false, "x": [] } }, "rationale_derived": "Floored at 2 by equistamp.06 (S.3; section 9): Regulator as client sets scope by tender.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by equistamp.06 (S.3; section 9): Regulator as client sets scope by tender.. No admissible signal caps it." }, "R": { "evaluator": "equistamp", "dimension": "R", "value": 2, "anchor": 2, "signals": [ "equistamp.07" ], "rationale": "Anchor 2 from the cited rows; low confidence, imported evidence.", "assessed": "2026-09-14", "assessor": "yohei/claude v0.4", "mechanism": "unknown", "graph_id": "assessment#648", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 2, "b": [ "equistamp.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "equistamp.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "equistamp.07" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "equistamp.07" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "equistamp.07" ] } }, "rationale_derived": "Capped at 2 by equistamp.07 (R.4): Deliverables go to the AI Office; no public reports found..", "rationale_leads": "Capped at 2 by equistamp.07 (R.4): Deliverables go to the AI Office; no public reports found.." }, "M": { "evaluator": "equistamp", "dimension": "M", "value": 2, "anchor": 2, "signals": [ "equistamp.08" ], "rationale": "Anchor 2 from the cited rows; low confidence, imported evidence.", "assessed": "2026-09-14", "assessor": "yohei/claude v0.4", "graph_id": "assessment#649", "evidence_tier": "tier 2 (ledger)", "derived": { "leads": { "v": 2, "b": [ "equistamp.08" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "equistamp.08" ] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "equistamp.08" ] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "equistamp.08" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "equistamp.08" ] } }, "rationale_derived": "Unevidenced under this policy: no admissible signal sets a bound. Not counted: equistamp.08 (no confirmed source (imported)).", "rationale_leads": "Capped at 2 by equistamp.08 (M.2): Methods not public.." }, "X": { "evaluator": "equistamp", "dimension": "X", "value": 3, "anchor": 3, "signals": [ "equistamp.09" ], "rationale": "Anchor 3 from the cited rows; low confidence, imported evidence. Evidence-limited: the cited sources are imported or unaudited only, so the value is held at 2 (the reading on the record would be 3) until a live source is confirmed.", "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#650", "evidence_tier": "tier 2 (ledger)", "derived": { "leads": { "v": 3, "b": [ "equistamp.09" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "equistamp.09" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "equistamp.09" ] }, "spans": { "v": 3, "b": [ "equistamp.09" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "equistamp.09" ] } }, "rationale_derived": "Floored at 3 by equistamp.09 (X.4): No lab-facing products found; clients are evaluators and governments.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by equistamp.09 (X.4): No lab-facing products found; clients are evaluators and governments.. No admissible signal caps it." } }, "signals": [ { "id": "equistamp.01", "evaluator": "equistamp", "dimension": "F", "direction": "for", "claim": "Revenue from evaluators and governments; leads EU Lot 3 (EUR 1.17M to consortium) and Lot 4 (EUR 0.88M).", "sources": [ "ted-864574", "evaluators-ledger", "nemesys-far-blog", "equistamp-substack" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "floor": 3 }, "rule": "F.10", "quote": "organisations that won contracts: EquiStamp, METR, Epoch AI, FAR.AI, SaferAI, SecureBio", "quote_source": "equistamp-substack", "graph_id": "signal#325", "source_graph_ids": [ "source#184", "source#81", "source#128", "source#72" ] }, { "id": "equistamp.02", "evaluator": "equistamp", "dimension": "F", "direction": "against", "claim": "Private company; no funding or ownership disclosed.", "sources": [ "evaluators-ledger", "equistamp-site" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 3 }, "rule": "F.10", "quote": "We're in the process of becoming a Public Benefit Corporation", "quote_source": "equistamp-site", "graph_id": "signal#326", "source_graph_ids": [ "source#81", "source#71" ] }, { "id": "equistamp.03", "evaluator": "equistamp", "dimension": "G", "direction": "against", "claim": "No COI policy on the site.", "sources": [ "evaluators-ledger", "equistamp-site" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 1 }, "rule": "G.2", "graph_id": "signal#327", "source_graph_ids": [ "source#81", "source#71" ] }, { "id": "equistamp.04", "evaluator": "equistamp", "dimension": "P", "direction": "against", "claim": "Founders' prior affiliations and any lab ties not established.", "sources": [ "evaluators-ledger" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 2 }, "rule": "P.6", "graph_id": "signal#328", "source_graph_ids": [ "source#81" ] }, { "id": "equistamp.05", "evaluator": "equistamp", "dimension": "A", "direction": "against", "claim": "Access is through client engagements; no pre-release lab access documented.", "sources": [ "evaluators-ledger", "equistamp-site" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 0 }, "rule": "A.6", "quote": "We build and run evaluations, baselines, and benchmarks", "quote_source": "equistamp-site", "graph_id": "signal#329", "source_graph_ids": [ "source#81", "source#71" ] }, { "id": "equistamp.06", "evaluator": "equistamp", "dimension": "S", "direction": "for", "claim": "Regulator as client sets scope by tender.", "sources": [ "ted-864574", "euaio-tender-call" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "floor": 2 }, "rule": "S.3; section 9", "quote": "technical assistance aimed at supporting the monitoring of compliance", "quote_source": "euaio-tender-call", "graph_id": "signal#330", "source_graph_ids": [ "source#184", "source#80" ] }, { "id": "equistamp.07", "evaluator": "equistamp", "dimension": "R", "direction": "against", "claim": "Deliverables go to the AI Office; no public reports found.", "sources": [ "ted-864574" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 2 }, "rule": "R.4", "graph_id": "signal#331", "source_graph_ids": [ "source#184" ] }, { "id": "equistamp.08", "evaluator": "equistamp", "dimension": "M", "direction": "against", "claim": "Methods not public.", "sources": [ "evaluators-ledger" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 2 }, "rule": "M.2", "graph_id": "signal#332", "source_graph_ids": [ "source#81" ] }, { "id": "equistamp.09", "evaluator": "equistamp", "dimension": "X", "direction": "for", "claim": "No lab-facing products found; clients are evaluators and governments.", "sources": [ "evaluators-ledger", "equistamp-site" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "floor": 3 }, "rule": "X.4", "quote": "EquiStamp provides evaluation implementation, data annotation, and project operations for AI safety labs", "quote_source": "equistamp-site", "graph_id": "signal#333", "source_graph_ids": [ "source#81", "source#71" ] } ], "scores": { "lab": 48, "regulator": 45, "public": 53, "equal": 50 }, "scores_by_policy": { "leads": { "lab": { "score": 49, "band": "conditional", "coverage": 8, "range": [ 49, 49 ] }, "regulator": { "score": 46, "band": "conditional", "coverage": 8, "range": [ 46, 46 ] }, "public": { "score": 53, "band": "conditional", "coverage": 8, "range": [ 53, 53 ] }, "equal": { "score": 50, "band": "conditional", "coverage": 8, "range": [ 50, 50 ] } }, "standard": { "lab": { "score": 48, "band": "conditional", "coverage": 6, "range": [ 39, 58 ] }, "regulator": { "score": 45, "band": "conditional", "coverage": 6, "range": [ 36, 56 ] }, "public": { "score": 53, "band": "conditional", "coverage": 6, "range": [ 43, 63 ] }, "equal": { "score": 50, "band": "conditional", "coverage": 6, "range": [ 38, 63 ] } }, "against_interest": { "lab": { "score": 47, "band": "conditional", "coverage": 5, "range": [ 36, 60 ] }, "regulator": { "score": 45, "band": "conditional", "coverage": 5, "range": [ 36, 56 ] }, "public": { "score": 52, "band": "conditional", "coverage": 5, "range": [ 39, 64 ] }, "equal": { "score": 45, "band": "conditional", "coverage": 5, "range": [ 28, 66 ] } }, "spans": { "lab": { "score": 51, "band": "conditional", "coverage": 4, "range": [ 30, 71 ] }, "regulator": { "score": 48, "band": "conditional", "coverage": 4, "range": [ 26, 71 ] }, "public": { "score": 64, "band": "conditional", "coverage": 4, "range": [ 29, 84 ] }, "equal": { "score": 56, "band": "conditional", "coverage": 4, "range": [ 28, 78 ] } }, "primary": { "lab": { "score": 63, "band": "clear", "coverage": 2, "range": [ 22, 88 ] }, "regulator": { "score": 61, "band": "clear", "coverage": 2, "range": [ 21, 86 ] }, "public": { "score": 68, "band": "clear", "coverage": 2, "range": [ 24, 89 ] }, "equal": { "score": 63, "band": "clear", "coverage": 2, "range": [ 16, 91 ] } } }, "band": "conditional", "coverage": 6, "floor": "G", "what_moves": { "lab": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 54, "band": "conditional", "base": 48 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 51, "band": "conditional", "base": 48 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 55, "band": "conditional", "base": 48 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 53, "band": "conditional", "base": 48 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 53, "band": "conditional", "base": 48 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 50, "band": "conditional", "base": 48 } ], "regulator": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 50, "band": "conditional", "base": 45 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 48, "band": "conditional", "base": 45 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 52, "band": "conditional", "base": 45 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 52, "band": "conditional", "base": 45 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 50, "band": "conditional", "base": 45 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 45, "band": "conditional", "base": 45 } ], "public": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 61, "band": "conditional", "base": 53 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 58, "band": "conditional", "base": 53 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 55, "band": "conditional", "base": 53 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 53 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 55, "band": "conditional", "base": 53 } ], "equal": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 54, "band": "conditional", "base": 50 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 54, "band": "conditional", "base": 50 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 54, "band": "conditional", "base": 50 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 54, "band": "conditional", "base": 50 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 54, "band": "conditional", "base": 50 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 54, "band": "conditional", "base": 50 } ] } }, { "id": "euaio", "name": "EU AI Office", "type": "gov", "hq": "Brussels, BE", "domains": [ "bio", "cyber", "misuse", "assurance" ], "confidence": "med", "summary": "Regulator behind the GPAI Code of Practice; external-evaluator obligations enforceable since August 2026.", "what_would_move_the_score": "An accreditation pathway for external evaluators and a public reporting schema.", "role": "government", "dissent": { "lower": "Scope (S) at 2 could be 1. The public record shows the Office has run no evaluation of any frontier model; the only scope it has exercised is a procurement of technical assistance whose deliverables are shaped by the Code that signatory labs themselves negotiated. Until the Office uses Article 92, the labs effectively define what is tested through their own Safety and Security Model Reports, which is the anchor-1 picture.", "higher": "Scope (S) at 2 could be 3. Article 92 gives the Office the legal power to evaluate any general-purpose model and to compel access, including source code, with fines for refusal, and enforcement has been live since August 2026. The Office set the six risk lots of the tender itself. A body that writes the questions and can compel answers is at least anchor 3; section 9 holds it down only for lack of a published run." }, "list_group": "government", "graph_id": "evaluator#211", "values": { "F": 3, "G": 3, "P": 3, "A": 3, "S": 4, "R": 2, "M": 2, "X": 4 }, "values_by_policy": { "leads": { "F": 3, "G": 3, "P": 3, "A": 3, "S": 4, "R": 2, "M": 2, "X": 4 }, "standard": { "F": 3, "G": 3, "P": 3, "A": 3, "S": 4, "R": 2, "M": 2, "X": 4 }, "against_interest": { "F": 3, "G": 3, "P": 3, "A": 3, "S": 4, "R": 2, "M": 2, "X": 4 }, "spans": { "F": 3, "G": 3, "P": 3, "A": 3, "S": 4, "R": 2, "M": 2, "X": 4 }, "primary": { "F": 3, "G": null, "P": null, "A": 3, "S": 4, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "euaio", "dimension": "F", "value": 3, "anchor": 3, "signals": [ "euaio.06", "euaio.10" ], "rationale": "Anchor 3 pending a confirmed bounded negative; funding is public by construction but the gate asks for the row.", "assessed": "2026-09-14", "assessor": "yohei/claude v0.4", "open_questions": [ "Add a confirmed bounded negative from a primary source to restore anchor 4." ], "graph_id": "assessment#651", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 3, "b": [ "euaio.10" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "euaio.10" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "euaio.10" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "euaio.10" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": 3, "b": [ "euaio.10" ], "h": false, "c": false, "u": false, "x": [ "euaio.06" ] } }, "rationale_derived": "Capped at 3 by euaio.10 (F.12): Public budget under the Digital Europe Programme; EUR 7.37M awarded across six lots to contracted evaluators. No confirmed negative on file.. Floors up to 3 from euaio.06 do not exceed the cap.", "rationale_leads": "Capped at 3 by euaio.10 (F.12): Public budget under the Digital Europe Programme; EUR 7.37M awarded across six lots to contracted evaluators. No confirmed negative on file.. Floors up to 3 from euaio.06 do not exceed the cap." }, "G": { "evaluator": "euaio", "dimension": "G", "value": 3, "anchor": 3, "signals": [ "euaio.07", "euaio.12" ], "rationale": "Anchor 3: public body with a statutory basis (euaio.07). No tier-1 source for an independent board or external review, so 4 is unearned under the tier rule (C10). Whether a formal COI policy is published is an open question.", "assessed": "2026-09-15", "assessor": "yohei/claude v0", "graph_id": "assessment#652", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 3, "b": [ "euaio.12" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "euaio.12" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "euaio.12" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "euaio.12" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "euaio.07", "euaio.12" ] } }, "rationale_derived": "Floored at 3 by euaio.12 (G.2): AI Office staff are Commission officials bound by the EU Staff Regulations, which require screening of actual or potential conflicts of interest (Article 11a) and restrict post-employment activity for two years (Article 16).. No admissible signal caps it.", "rationale_leads": "Floored at 3 by euaio.12 (G.2): AI Office staff are Commission officials bound by the EU Staff Regulations, which require screening of actual or potential conflicts of interest (Article 11a) and restrict post-employment activity for two years (Article 16).. No admissible signal caps it." }, "P": { "evaluator": "euaio", "dimension": "P", "value": 3, "anchor": 3, "signals": [ "euaio.08" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#653", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 3, "b": [ "euaio.08" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "euaio.08" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "euaio.08" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "euaio.08" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "euaio.08" ] } }, "rationale_derived": "Floored at 3 by euaio.08 (P.5): Civil-service conflict rules.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by euaio.08 (P.5): Civil-service conflict rules.. No admissible signal caps it." }, "A": { "evaluator": "euaio", "dimension": "A", "value": 3, "anchor": 3, "signals": [ "euaio.01" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "mechanism": "statutory", "graph_id": "assessment#654", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 3, "b": [ "euaio.01" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "euaio.01" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "euaio.01" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "euaio.01" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": 3, "b": [ "euaio.01" ], "h": false, "c": false, "u": false, "x": [] } }, "rationale_derived": "Floored at 3 by euaio.01 (A.2): Legal power to compel; the only body whose access does not depend on goodwill.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by euaio.01 (A.2): Legal power to compel; the only body whose access does not depend on goodwill.. No admissible signal caps it." }, "S": { "evaluator": "euaio", "dimension": "S", "value": 4, "anchor": 4, "signals": [ "euaio.02", "euaio.03", "euaio.11", "euaio.13" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#655", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 4, "b": [ "euaio.13" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 4, "b": [ "euaio.13" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 4, "b": [ "euaio.13" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 4, "b": [ "euaio.13" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": 4, "b": [ "euaio.13" ], "h": false, "c": false, "u": false, "x": [ "euaio.02", "euaio.03" ] } }, "rationale_derived": "Floored at 4 by euaio.13 (S.4): The AI Office holds the legal power under the AI Act to evaluate general-purpose models with systemic risk and to compel access and information; scope for those evaluations is set by the regulator, not the developer.. No admissible signal caps it.", "rationale_leads": "Floored at 4 by euaio.13 (S.4): The AI Office holds the legal power under the AI Act to evaluate general-purpose models with systemic risk and to compel access and information; scope for those evaluations is set by the regulator, not the developer.. No admissible signal caps it." }, "R": { "evaluator": "euaio", "dimension": "R", "value": 2, "anchor": 2, "signals": [ "euaio.04" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "mechanism": "statutory", "graph_id": "assessment#656", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "euaio.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "euaio.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "euaio.04" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "euaio.04" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "euaio.04" ] } }, "rationale_derived": "Capped at 2 by euaio.04 (R.4): Safety and Security Model Reports are submitted to the Office, not the public..", "rationale_leads": "Capped at 2 by euaio.04 (R.4): Safety and Security Model Reports are submitted to the Office, not the public.." }, "M": { "evaluator": "euaio", "dimension": "M", "value": 2, "anchor": 2, "signals": [ "euaio.05" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#657", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 2, "b": [ "euaio.05" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "euaio.05" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "euaio.05" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "euaio.05" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "euaio.05" ] } }, "rationale_derived": "Capped at 2 by euaio.05 (M.2): No accreditation pathway or public reporting schema yet; thin evaluator bench..", "rationale_leads": "Capped at 2 by euaio.05 (M.2): No accreditation pathway or public reporting schema yet; thin evaluator bench.." }, "X": { "evaluator": "euaio", "dimension": "X", "value": 4, "anchor": 4, "signals": [ "euaio.09" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#658", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 4, "b": [ "euaio.09" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 4, "b": [ "euaio.09" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 4, "b": [ "euaio.09" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 4, "b": [ "euaio.09" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "euaio.09" ] } }, "rationale_derived": "Floored at 4 by euaio.09 (X.7): No commercial products.. No admissible signal caps it.", "rationale_leads": "Floored at 4 by euaio.09 (X.7): No commercial products.. No admissible signal caps it." } }, "signals": [ { "id": "euaio.01", "evaluator": "euaio", "dimension": "A", "direction": "for", "claim": "Legal power to compel; the only body whose access does not depend on goodwill.", "sources": [ "metr-regs", "euaio-ai-act-92" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "A.2", "quote": "may request access to the general-purpose AI model concerned through APIs or further appropriate technical means", "quote_source": "euaio-ai-act-92", "graph_id": "signal#334", "source_graph_ids": [ "source#118", "source#73" ] }, { "id": "euaio.02", "evaluator": "euaio", "dimension": "S", "direction": "for", "claim": "Contracts its own evaluators (FAR.AI-led consortium) rather than relying on lab-chosen ones.", "sources": [ "longtermwiki-far" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "S.3", "quote": "FAR AI was selected by the European Commission's AI Office to lead CBRN risk research under tender EC-CNECT/2025/OP/0032", "quote_source": "longtermwiki-far", "graph_id": "signal#335", "source_graph_ids": [ "source#106" ] }, { "id": "euaio.03", "evaluator": "euaio", "dimension": "S", "direction": "for", "claim": "Code sets the access floor: helpful-only versions where security allows, at least 20 business days, external evaluators for each new frontier model.", "sources": [ "metr-regs", "arxiv-access-taxonomy" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "S.4", "quote": "A developer should engage independent external evaluators for each new frontier model", "quote_source": "metr-regs", "graph_id": "signal#336", "source_graph_ids": [ "source#118", "source#36" ] }, { "id": "euaio.04", "evaluator": "euaio", "dimension": "R", "direction": "against", "claim": "Safety and Security Model Reports are submitted to the Office, not the public.", "sources": [ "metr-regs", "euaio-cop-text" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "R.4", "quote": "Signatories commit to reporting to the AI Office information about their model", "quote_source": "euaio-cop-text", "graph_id": "signal#337", "source_graph_ids": [ "source#118", "source#76" ] }, { "id": "euaio.05", "evaluator": "euaio", "dimension": "M", "direction": "against", "claim": "No accreditation pathway or public reporting schema yet; thin evaluator bench.", "sources": [ "techpolicy-aef1" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "M.2", "quote": "the pool of qualified evaluators remains thin", "quote_source": "techpolicy-aef1", "graph_id": "signal#338", "source_graph_ids": [ "source#180" ] }, { "id": "euaio.06", "evaluator": "euaio", "dimension": "F", "direction": "for", "claim": "Public funding.", "sources": [ "metr-regs", "euaio-office-decision" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "F.9", "quote": "established within the Commission as part of the administrative structure of the Directorate-General", "quote_source": "euaio-office-decision", "graph_id": "signal#339", "source_graph_ids": [ "source#118", "source#78" ] }, { "id": "euaio.07", "evaluator": "euaio", "dimension": "G", "direction": "for", "claim": "Regulator with statutory basis.", "sources": [ "metr-regs", "euaio-ai-office-page" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "G.1", "quote": "established within the European Commission as the foundation for a single AI governance system", "quote_source": "euaio-ai-office-page", "graph_id": "signal#340", "source_graph_ids": [ "source#118", "source#74" ] }, { "id": "euaio.08", "evaluator": "euaio", "dimension": "P", "direction": "for", "claim": "Civil-service conflict rules.", "sources": [ "metr-regs", "euaio-ceo-staffregs", "euaio-staff-regs" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "P.5", "quote": "Article 16 of the old staff regulations included a two-year notification period after staff leave their EU job", "quote_source": "euaio-ceo-staffregs", "graph_id": "signal#341", "source_graph_ids": [ "source#118", "source#75", "source#79" ] }, { "id": "euaio.09", "evaluator": "euaio", "dimension": "X", "direction": "for", "claim": "No commercial products.", "sources": [ "metr-regs", "euaio-ai-office-page", "euaio-newsletter-77" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 4 }, "rule": "X.7", "quote": "It also enforces the rules for GPAI models and supports the governance bodies in Member States", "quote_source": "euaio-ai-office-page", "graph_id": "signal#342", "source_graph_ids": [ "source#118", "source#74", "source#77" ] }, { "id": "euaio.10", "evaluator": "euaio", "dimension": "F", "direction": "against", "claim": "Public budget under the Digital Europe Programme; EUR 7.37M awarded across six lots to contracted evaluators. No confirmed negative on file.", "sources": [ "ted-864574", "euaio-newsletter-77" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 3 }, "rule": "F.12", "quote": "The €9,080,000 tender is divided into six lots", "quote_source": "euaio-newsletter-77", "graph_id": "signal#343", "source_graph_ids": [ "source#184", "source#77" ] }, { "id": "euaio.11", "evaluator": "euaio", "dimension": "S", "direction": "for", "claim": "Contract notice TED 864574-2025, Technical Assistance for AI Safety, published 26 December 2025; contractors self-report include METR, FAR.AI, SecureBio, SaferAI and Epoch.", "sources": [ "ted-864574-notice", "ted-864574", "euaio-tender-call", "euaio-newsletter-77" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2025-12-26", "bound": { "floor": 3 }, "rule": "S.3", "quote": "This tender has been split into six lots", "quote_source": "euaio-tender-call", "graph_id": "signal#344", "source_graph_ids": [ "source#183", "source#184", "source#80", "source#77" ] }, { "id": "euaio.12", "evaluator": "euaio", "dimension": "G", "direction": "for", "claim": "AI Office staff are Commission officials bound by the EU Staff Regulations, which require screening of actual or potential conflicts of interest (Article 11a) and restrict post-employment activity for two years (Article 16).", "sources": [ "euaio-ceo-staffregs", "euaio-staff-regs" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "floor": 3 }, "rule": "G.2", "quote": "a concrete procedure for properly screening new staff for actual or potential conflicts of interest", "quote_source": "euaio-ceo-staffregs", "note": "Statutory civil-service conflict rules count as a conflict policy for a public body (G.2): floor 3. G.7 four needs tier-1 evidence of an independent board and external review; none.", "graph_id": "signal#345", "source_graph_ids": [ "source#75", "source#79" ] }, { "id": "euaio.13", "evaluator": "euaio", "dimension": "S", "direction": "for", "claim": "The AI Office holds the legal power under the AI Act to evaluate general-purpose models with systemic risk and to compel access and information; scope for those evaluations is set by the regulator, not the developer.", "sources": [ "metr-regs", "euaio-ai-act-92" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "as_of": "2026-09-15", "bound": { "floor": 4 }, "rule": "S.4", "quote": "may request access to the general-purpose AI model concerned through APIs or further appropriate technical means", "quote_source": "euaio-ai-act-92", "graph_id": "signal#346", "source_graph_ids": [ "source#118", "source#73" ] } ], "scores": { "lab": 75, "regulator": 74, "public": 73, "equal": 75 }, "scores_by_policy": { "leads": { "lab": { "score": 75, "band": "clear", "coverage": 8, "range": [ 75, 75 ] }, "regulator": { "score": 74, "band": "clear", "coverage": 8, "range": [ 74, 74 ] }, "public": { "score": 73, "band": "clear", "coverage": 8, "range": [ 73, 73 ] }, "equal": { "score": 75, "band": "clear", "coverage": 8, "range": [ 75, 75 ] } }, "standard": { "lab": { "score": 75, "band": "clear", "coverage": 8, "range": [ 75, 75 ] }, "regulator": { "score": 74, "band": "clear", "coverage": 8, "range": [ 74, 74 ] }, "public": { "score": 73, "band": "clear", "coverage": 8, "range": [ 73, 73 ] }, "equal": { "score": 75, "band": "clear", "coverage": 8, "range": [ 75, 75 ] } }, "against_interest": { "lab": { "score": 75, "band": "clear", "coverage": 8, "range": [ 75, 75 ] }, "regulator": { "score": 74, "band": "clear", "coverage": 8, "range": [ 74, 74 ] }, "public": { "score": 73, "band": "clear", "coverage": 8, "range": [ 73, 73 ] }, "equal": { "score": 75, "band": "clear", "coverage": 8, "range": [ 75, 75 ] } }, "spans": { "lab": { "score": 75, "band": "clear", "coverage": 8, "range": [ 75, 75 ] }, "regulator": { "score": 74, "band": "clear", "coverage": 8, "range": [ 74, 74 ] }, "public": { "score": 73, "band": "clear", "coverage": 8, "range": [ 73, 73 ] }, "equal": { "score": 75, "band": "clear", "coverage": 8, "range": [ 75, 75 ] } }, "primary": { "lab": { "score": 82, "band": "clear", "coverage": 3, "range": [ 45, 91 ] }, "regulator": { "score": 84, "band": "clear", "coverage": 3, "range": [ 46, 91 ] }, "public": { "score": 81, "band": "clear", "coverage": 3, "range": [ 33, 93 ] }, "equal": { "score": 83, "band": "clear", "coverage": 3, "range": [ 31, 94 ] } } }, "band": "clear", "coverage": 8, "floor": "R", "what_moves": { "lab": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 79, "band": "clear", "base": 75 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 77, "band": "clear", "base": 75 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 77, "band": "clear", "base": 75 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 80, "band": "clear", "base": 75 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 78, "band": "clear", "base": 75 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 77, "band": "clear", "base": 75 } ], "regulator": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 74 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 76, "band": "clear", "base": 74 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 76, "band": "clear", "base": 74 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 79, "band": "clear", "base": 74 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 78, "band": "clear", "base": 74 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 76, "band": "clear", "base": 74 } ], "public": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 79, "band": "clear", "base": 73 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 76, "band": "clear", "base": 73 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 76, "band": "clear", "base": 73 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 74, "band": "clear", "base": 73 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 78, "band": "clear", "base": 73 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 74, "band": "clear", "base": 73 } ], "equal": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 75 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 75 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 75 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 75 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 78, "band": "clear", "base": 75 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 78, "band": "clear", "base": 75 } ] } }, { "id": "farai", "name": "FAR.AI", "type": "nonprofit", "hq": "Berkeley, US", "domains": [ "jailbreak", "bio", "cyber", "misuse" ], "confidence": "high", "summary": "Red-teaming and adversarial-robustness nonprofit; leads the EU AI Office CBRN risk-modelling contract.", "what_would_move_the_score": "Disclose the AI Safety Fund share of budget and whether lab-funded pools are excluded from evaluation work.", "role": "referee", "dissent": { "lower": "Access could be 0. No cited source records the terms of any lab engagement: the wiki says FAR.AI began red-teaming leading models for frontier labs in late 2023 and that publicly confirmed engagements include OpenAI, but neither it nor the $30M post shows pre-release checkpoints, safeguards off, or logs. Anchor 0 is public API only, and that is all the record demonstrates; the EU contract is a regulator engagement, not lab access.", "higher": "Access could be 2. OpenAI's GPT-5 system card credits FAR.AI with 80 hours of red-teaming before release, and pre-release red-teamers typically test early checkpoints under relaxed safeguards; the transparency page describes engagements with companies building frontier models under NDA. If the engagement terms show safeguards off or extended windows, anchor 2 applies, and helpful-only or log access would reach 3." }, "list_group": "referee", "graph_id": "evaluator#212", "values": { "F": 3, "G": 2, "P": 2, "A": 1, "S": 3, "R": 2, "M": 2, "X": 2 }, "values_by_policy": { "leads": { "F": 3, "G": 2, "P": 2, "A": 1, "S": 3, "R": 2, "M": 2, "X": 2 }, "standard": { "F": 3, "G": 2, "P": 2, "A": 1, "S": 3, "R": 2, "M": 2, "X": 2 }, "against_interest": { "F": 3, "G": null, "P": 2, "A": 1, "S": 3, "R": null, "M": null, "X": 2 }, "spans": { "F": 3, "G": 2, "P": null, "A": 1, "S": 3, "R": 2, "M": 2, "X": 2 }, "primary": { "F": null, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "farai", "dimension": "F", "value": 3, "anchor": 3, "signals": [ "farai.01", "farai.02", "farai.10", "farai.12", "farai.14", "farai.17", "farai.16" ], "assessed": "2026-09-15", "assessor": "yohei/claude v1.0", "graph_id": "assessment#659", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 3, "b": [ "farai.02", "farai.12", "farai.17", "farai.16" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "farai.02", "farai.12", "farai.17", "farai.16" ], "h": false, "c": false, "u": false, "x": [ "farai.14" ] }, "against_interest": { "v": 3, "b": [ "farai.02", "farai.12", "farai.17", "farai.16" ], "h": false, "c": false, "u": false, "x": [ "farai.01", "farai.10", "farai.14" ] }, "spans": { "v": 3, "b": [ "farai.02", "farai.12", "farai.17", "farai.16" ], "h": false, "c": false, "u": false, "x": [ "farai.14" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "farai.01", "farai.02", "farai.10", "farai.12", "farai.14", "farai.16", "farai.17" ] } }, "rationale_derived": "Capped at 3 by farai.02 (F.6): Also funded by the AI Safety Fund, financed by the Frontier Model Forum, the labs' industry body.; capped at 3 by farai.12 (F.6): Largest single grant: $28,675,000 over three years from Coefficient Giving (September 2025); over $30 million in 2025 commitments from Coefficient, Schmidt Sciences, SFF, CSET and the Frontier Model Forum's AI Safety Fund.; capped at 3 by farai.17 (F.6): Founders Pledge, a donor-advised giving vehicle, granted $500,000 to the FAR.AI Integrity Team for EU AI Office manipulation evaluations.; capped at 3 by farai.16 (F.3): FAR.AI charges for-profit AI developers market rates for consulting and caps that revenue at 10% of total annual revenue; the transparency page reports TY2024 earned revenue of $429K against $24.3M total.. Floors up to 3 from farai.01, farai.10 do not exceed the cap. Not counted under this policy: farai.14 (no confirmed source (unaudited)).", "rationale_leads": "Capped at 3 by farai.02 (F.6): Also funded by the AI Safety Fund, financed by the Frontier Model Forum, the labs' industry body.; capped at 3 by farai.12 (F.6): Largest single grant: $28,675,000 over three years from Coefficient Giving (September 2025); over $30 million in 2025 commitments from Coefficient, Schmidt Sciences, SFF, CSET and the Frontier Model Forum's AI Safety Fund.; capped at 3 by farai.17 (F.6): Founders Pledge, a donor-advised giving vehicle, granted $500,000 to the FAR.AI Integrity Team for EU AI Office manipulation evaluations.; capped at 3 by farai.16 (F.3): FAR.AI charges for-profit AI developers market rates for consulting and caps that revenue at 10% of total annual revenue; the transparency page reports TY2024 earned revenue of $429K against $24.3M total.. Floors up to 3 from farai.01, farai.10, farai.14 do not exceed the cap." }, "G": { "evaluator": "farai", "dimension": "G", "value": 2, "anchor": 2, "signals": [ "farai.05" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#660", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "farai.05" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "farai.05" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "farai.05" ] }, "spans": { "v": 2, "b": [ "farai.05" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "farai.05" ] } }, "rationale_derived": "Floored at 2 by farai.05 (G.1): Nonprofit with diversified disclosed funders.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by farai.05 (G.1): Nonprofit with diversified disclosed funders.. No admissible signal caps it." }, "P": { "evaluator": "farai", "dimension": "P", "value": 2, "anchor": 2, "signals": [ "farai.07" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#661", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 2, "b": [ "farai.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "farai.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "farai.07" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "farai.07" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "farai.07" ] } }, "rationale_derived": "Capped at 2 by farai.07 (P.3): Runs field-building programs that place researchers into labs..", "rationale_leads": "Capped at 2 by farai.07 (P.3): Runs field-building programs that place researchers into labs.." }, "A": { "evaluator": "farai", "dimension": "A", "value": 1, "anchor": 1, "signals": [ "farai.06" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "lab-controlled", "graph_id": "assessment#662", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 1, "b": [ "farai.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "farai.06" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "farai.06" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "farai.06" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "farai.06" ] } }, "rationale_derived": "Floored at 1 by farai.06 (A.1): Pre- and post-deployment red teaming for labs and EU bodies.. No admissible signal caps it.", "rationale_leads": "Floored at 1 by farai.06 (A.1): Pre- and post-deployment red teaming for labs and EU bodies.. No admissible signal caps it." }, "S": { "evaluator": "farai", "dimension": "S", "value": 3, "anchor": 3, "signals": [ "farai.03", "farai.11", "farai.13" ], "assessed": "2026-09-15", "assessor": "yohei/claude v1.0", "graph_id": "assessment#663", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 3, "b": [ "farai.03", "farai.11" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "farai.03", "farai.11" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "farai.03", "farai.11" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "farai.03" ], "h": false, "c": false, "u": false, "x": [ "farai.11" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "farai.03", "farai.11", "farai.13" ] } }, "rationale_derived": "Floored at 3 by farai.03 (S.3): Selected by the European Commission to lead a three-year CBRN technical-assistance contract, with a regulator as client.; floored at 3 by farai.11 (S.3): Leads EU AI Office Lot 1 (CBRN) with SecureBio and SaferAI under a three-year contract, EUR 1.43M to the consortium.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by farai.03 (S.3): Selected by the European Commission to lead a three-year CBRN technical-assistance contract, with a regulator as client.; floored at 3 by farai.11 (S.3): Leads EU AI Office Lot 1 (CBRN) with SecureBio and SaferAI under a three-year contract, EUR 1.43M to the consortium.. No admissible signal caps it." }, "R": { "evaluator": "farai", "dimension": "R", "value": 2, "anchor": 2, "signals": [ "farai.04" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "self-imposed", "graph_id": "assessment#664", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "farai.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "farai.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "farai.04" ] }, "spans": { "v": 2, "b": [ "farai.04" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "farai.04" ] } }, "rationale_derived": "Floored at 2 by farai.04 (R.7): Publishes jailbreak and stress-test results across vendors.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by farai.04 (R.7): Publishes jailbreak and stress-test results across vendors.. No admissible signal caps it." }, "M": { "evaluator": "farai", "dimension": "M", "value": 2, "anchor": 2, "signals": [ "farai.08" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#665", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "farai.08" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "farai.08" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "farai.08" ] }, "spans": { "v": 2, "b": [ "farai.08" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "farai.08" ] } }, "rationale_derived": "Floored at 2 by farai.08 (M.2): Publishes methods and papers.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by farai.08 (M.2): Publishes methods and papers.. No admissible signal caps it." }, "X": { "evaluator": "farai", "dimension": "X", "value": 2, "anchor": 2, "signals": [ "farai.09", "farai.15" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "resolution": { "rule": "X.3", "value": 2, "note": "FAR.AI's transparency page records paid work for for-profit frontier developers at market rates; paid consulting for a lab caps role incompatibility at 2 whatever else is open or free." }, "graph_id": "assessment#666", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "farai.15" ], "h": false, "c": true, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "farai.15" ], "h": false, "c": true, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "farai.15" ], "h": false, "c": true, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "farai.15" ], "h": false, "c": true, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "farai.09", "farai.15" ] } }, "rationale_derived": "Conflict: floor 3 from farai.09 (X.4) against cap 2 from farai.15 (X.3); resolved at 2 by X.3: FAR.AI's transparency page records paid work for for-profit frontier developers at market rates; paid consulting for a lab caps role incompatibility at 2 whatever else is open or free.", "rationale_leads": "Conflict: floor 3 from farai.09 (X.4) against cap 2 from farai.15 (X.3); resolved at 2 by X.3: FAR.AI's transparency page records paid work for for-profit frontier developers at market rates; paid consulting for a lab caps role incompatibility at 2 whatever else is open or free." } }, "signals": [ { "id": "farai.01", "evaluator": "farai", "dimension": "F", "direction": "for", "claim": "More than $30M in 2025 commitments from Coefficient Giving, Schmidt Sciences, SFF and CSET.", "sources": [ "far-30m" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "F.12", "quote": "FAR.AI has secured over $30 million in funding commitments throughout 2025", "quote_source": "far-30m", "graph_id": "signal#347", "source_graph_ids": [ "source#82" ] }, { "id": "farai.02", "evaluator": "farai", "dimension": "F", "direction": "against", "claim": "Also funded by the AI Safety Fund, financed by the Frontier Model Forum, the labs' industry body.", "sources": [ "far-30m" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 3 }, "rule": "F.6", "quote": "the AI Safety Fund (AISF), supported by the Frontier Model Forum (FMF)", "quote_source": "far-30m", "graph_id": "signal#348", "source_graph_ids": [ "source#82" ] }, { "id": "farai.03", "evaluator": "farai", "dimension": "S", "direction": "for", "claim": "Selected by the European Commission to lead a three-year CBRN technical-assistance contract, with a regulator as client.", "sources": [ "longtermwiki-far" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "S.3", "quote": "FAR AI was selected by the European Commission's AI Office to lead Lot 1 (CBRN Risk Modelling and Evaluation)", "quote_source": "longtermwiki-far", "graph_id": "signal#349", "source_graph_ids": [ "source#106" ] }, { "id": "farai.04", "evaluator": "farai", "dimension": "R", "direction": "for", "claim": "Publishes jailbreak and stress-test results across vendors.", "sources": [ "far-30m" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "R.7", "quote": "we've published groundbreaking research on adversarial robustness, interpretability, and red-teaming", "quote_source": "far-30m", "graph_id": "signal#350", "source_graph_ids": [ "source#82" ] }, { "id": "farai.05", "evaluator": "farai", "dimension": "G", "direction": "for", "claim": "Nonprofit with diversified disclosed funders.", "sources": [ "far-30m" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "G.1", "quote": "FAR.AI is an independent nonprofit research organization", "quote_source": "far-30m", "graph_id": "signal#351", "source_graph_ids": [ "source#82" ] }, { "id": "farai.06", "evaluator": "farai", "dimension": "A", "direction": "for", "claim": "Pre- and post-deployment red teaming for labs and EU bodies.", "sources": [ "longtermwiki-far" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 1 }, "rule": "A.1", "quote": "FAR AI began red-teaming leading language models for frontier labs in Q4 2023, including red-teaming of GPT-4", "quote_source": "longtermwiki-far", "graph_id": "signal#352", "source_graph_ids": [ "source#106" ] }, { "id": "farai.07", "evaluator": "farai", "dimension": "P", "direction": "against", "claim": "Runs field-building programs that place researchers into labs.", "sources": [ "longtermwiki-far" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "P.3", "graph_id": "signal#353", "source_graph_ids": [ "source#106" ] }, { "id": "farai.08", "evaluator": "farai", "dimension": "M", "direction": "for", "claim": "Publishes methods and papers.", "sources": [ "far-30m" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "M.2", "quote": "we've published groundbreaking research on adversarial robustness, interpretability, and red-teaming", "quote_source": "far-30m", "graph_id": "signal#354", "source_graph_ids": [ "source#82" ] }, { "id": "farai.09", "evaluator": "farai", "dimension": "X", "direction": "for", "claim": "No commercial products; consulting limited to governments.", "sources": [ "longtermwiki-far" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "X.4", "quote": "FAR AI conducts empirical technical AI safety research without developing or deploying AI products of its own", "quote_source": "longtermwiki-far", "graph_id": "signal#355", "source_graph_ids": [ "source#106" ] }, { "id": "farai.10", "evaluator": "farai", "dimension": "F", "direction": "for", "claim": "Published cap: revenue from for-profit AI developers limited to 10% of annual revenue, charged at market rates with publication rights retained; TY2024 earned revenue $429K of $24.3M.", "sources": [ "far-transparency" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "quote": "is capped at a maximum of 10% of FAR.AI's total annual revenue", "bound": { "floor": 3 }, "rule": "F.3", "quote_source": "far-transparency", "graph_id": "signal#356", "source_graph_ids": [ "source#83" ] }, { "id": "farai.11", "evaluator": "farai", "dimension": "S", "direction": "for", "claim": "Leads EU AI Office Lot 1 (CBRN) with SecureBio and SaferAI under a three-year contract, EUR 1.43M to the consortium.", "sources": [ "ted-864574" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "floor": 3 }, "rule": "S.3", "graph_id": "signal#357", "source_graph_ids": [ "source#184" ] }, { "id": "farai.12", "evaluator": "farai", "dimension": "F", "direction": "against", "claim": "Largest single grant: $28,675,000 over three years from Coefficient Giving (September 2025); over $30 million in 2025 commitments from Coefficient, Schmidt Sciences, SFF, CSET and the Frontier Model Forum's AI Safety Fund.", "sources": [ "op-far-general-support", "far-30m" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2025-09", "quote": "FAR.AI has secured over $30 million in funding commitments throughout 2025", "bound": { "cap": 3 }, "rule": "F.6", "quote_source": "far-30m", "graph_id": "signal#358", "source_graph_ids": [ "source#137", "source#82" ] }, { "id": "farai.13", "evaluator": "farai", "dimension": "S", "direction": "for", "claim": "Founders Pledge granted $500,000 to the FAR.AI Integrity Team for EU AI Office manipulation evaluations.", "sources": [ "founderspledge-frontier" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2026", "bound": null, "rule": "S.3", "bound_note": "Section 9 (mis-dimensioned facts): a grant is a funding fact; informational on S and re-recorded on F as farai.17 (F.6). The regulator-client fact is already carried by farai.03 and farai.11.", "quote": "The grant funds the evaluations FAR.AI is now developing for the EU AI Office's consortium on manipulation", "quote_source": "founderspledge-frontier", "graph_id": "signal#359", "source_graph_ids": [ "source#88" ] }, { "id": "farai.14", "evaluator": "farai", "dimension": "F", "direction": "for", "claim": "A third-party wiki records a 10% cap on for-profit developer revenue. FY2024 revenue of $24.3M against $8.6M expenses is per the same source; span not yet captured.", "sources": [ "longtermwiki-far-cap" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2026-09-15", "quote": "caps revenue from for-profit AI developers at 10% of total annual revenue", "bound": { "floor": 3 }, "rule": "F.3", "quote_source": "longtermwiki-far-cap", "graph_id": "signal#360", "source_graph_ids": [ "source#105" ] }, { "id": "farai.15", "evaluator": "farai", "dimension": "X", "direction": "against", "claim": "FAR.AI's transparency page publishes a consulting policy whose clients include companies building frontier AI models, charged at market rates, with revenue from for-profit developers capped at 10% of annual revenue; TY2024 earned revenue was $429K.", "sources": [ "far-transparency" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 2 }, "rule": "X.3", "quote": "we charge market rates to for-profit developers to avoid subsidizing private actors", "quote_source": "far-transparency", "note": "X.3: paid consultation or framework work for a lab caps at 2, and consulting clients on a transparency page count. Tension noted with section 10 (lab fees for the evaluation itself change funding, not class); the page frames this as consulting, not evaluation.", "graph_id": "signal#361", "source_graph_ids": [ "source#83" ] }, { "id": "farai.17", "evaluator": "farai", "dimension": "F", "direction": "against", "claim": "Founders Pledge, a donor-advised giving vehicle, granted $500,000 to the FAR.AI Integrity Team for EU AI Office manipulation evaluations.", "sources": [ "founderspledge-frontier" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "F.6", "quote": "FAR.AI Integrity Team ($500,000)", "quote_source": "founderspledge-frontier", "note": "F.6: grants through donor-advised funds whose underlying donors are not public cap at 3. Re-recorded from farai.13 under section 9.", "graph_id": "signal#362", "source_graph_ids": [ "source#88" ] }, { "id": "farai.16", "evaluator": "farai", "dimension": "F", "direction": "against", "claim": "FAR.AI charges for-profit AI developers market rates for consulting and caps that revenue at 10% of total annual revenue; the transparency page reports TY2024 earned revenue of $429K against $24.3M total.", "sources": [ "far-transparency" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "F.3", "quote": "is capped at a maximum of 10% of FAR.AI's total annual revenue", "quote_source": "far-transparency", "note": "F.3: labs paying per engagement caps at 2, lifted to 3 by a published 10% cap at market rates with publication rights retained. Recorded as the against-side of farai.10 so the cap exists in the derivation.", "graph_id": "signal#363", "source_graph_ids": [ "source#83" ] } ], "scores": { "lab": 54, "regulator": 54, "public": 57, "equal": 53 }, "scores_by_policy": { "leads": { "lab": { "score": 54, "band": "conditional", "coverage": 8, "range": [ 54, 54 ] }, "regulator": { "score": 54, "band": "conditional", "coverage": 8, "range": [ 54, 54 ] }, "public": { "score": 57, "band": "conditional", "coverage": 8, "range": [ 57, 57 ] }, "equal": { "score": 53, "band": "conditional", "coverage": 8, "range": [ 53, 53 ] } }, "standard": { "lab": { "score": 54, "band": "conditional", "coverage": 8, "range": [ 54, 54 ] }, "regulator": { "score": 54, "band": "conditional", "coverage": 8, "range": [ 54, 54 ] }, "public": { "score": 57, "band": "conditional", "coverage": 8, "range": [ 57, 57 ] }, "equal": { "score": 53, "band": "conditional", "coverage": 8, "range": [ 53, 53 ] } }, "against_interest": { "lab": { "score": 55, "band": "conditional", "coverage": 5, "range": [ 38, 69 ] }, "regulator": { "score": 56, "band": "conditional", "coverage": 5, "range": [ 36, 71 ] }, "public": { "score": 63, "band": "conditional", "coverage": 5, "range": [ 38, 78 ] }, "equal": { "score": 55, "band": "conditional", "coverage": 5, "range": [ 34, 72 ] } }, "spans": { "lab": { "score": 54, "band": "conditional", "coverage": 7, "range": [ 49, 59 ] }, "regulator": { "score": 54, "band": "conditional", "coverage": 7, "range": [ 49, 59 ] }, "public": { "score": 59, "band": "conditional", "coverage": 7, "range": [ 50, 65 ] }, "equal": { "score": 54, "band": "conditional", "coverage": 7, "range": [ 47, 59 ] } }, "primary": { "lab": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "regulator": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "public": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "equal": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] } } }, "band": "conditional", "coverage": 8, "floor": "A", "what_moves": { "lab": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 58, "band": "conditional", "base": 54 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 54 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 54 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 59, "band": "clear", "base": 54 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 57, "band": "conditional", "base": 54 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 57, "band": "conditional", "base": 54 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 54 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 55, "band": "conditional", "base": 54 } ], "regulator": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 57, "band": "conditional", "base": 54 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 54 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 54 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 59, "band": "clear", "base": 54 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 59, "band": "conditional", "base": 54 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 57, "band": "conditional", "base": 54 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 54 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 54, "band": "conditional", "base": 54 } ], "public": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 64, "band": "conditional", "base": 57 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 61, "band": "conditional", "base": 57 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 61, "band": "conditional", "base": 57 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 59, "band": "clear", "base": 57 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 60, "band": "conditional", "base": 57 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 63, "band": "conditional", "base": 57 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 57 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 57 } ], "equal": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 56, "band": "clear", "base": 53 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 } ] } }, { "id": "grayswan", "name": "Gray Swan", "type": "vc", "hq": "Pittsburgh, US", "domains": [ "jailbreak", "cyber", "misuse" ], "confidence": "high", "summary": "Crowdsourced adversarial testing (Arena) plus Shade and Cygnal defense products; cited in 11 recent system cards.", "what_would_move_the_score": "Structural separation of the evaluation business, and independent governance for the OpenAI-related conflict beyond personal recusal.", "role": "vendor", "dissent": { "lower": "Personnel (1 to 0). The co-founder and chief scientist chairs OpenAI's Safety and Security Committee, which oversees the very releases in which Gray Swan is cited. The only evidence of recusal is one 2024 press sentence describing a personal arrangement; no written policy, scope, or administrator is public. P.1 gives 0 where a recusal is not documented, and a single unverifiable press line is thin documentation.", "higher": "Role incompatibility (0 to 1). The record documents that labs use Gray Swan's platform and are named in its system-card citations, but not that any lab buys Cygnal, the runtime defense product; lab use may be limited to Arena and Shade, which are testing services. X.1's 0 requires a defense or monitoring product sold to an evaluated lab; on the documented facts, anchor 1 (services to labs) fits." }, "list_group": "commercial", "graph_id": "evaluator#213", "values": { "F": 1, "G": 1, "P": 1, "A": 2, "S": 2, "R": 2, "M": 2, "X": 0 }, "values_by_policy": { "leads": { "F": 1, "G": 1, "P": 1, "A": 2, "S": 2, "R": 2, "M": 2, "X": 0 }, "standard": { "F": 1, "G": 1, "P": 1, "A": 2, "S": 2, "R": 2, "M": 2, "X": 0 }, "against_interest": { "F": 1, "G": 1, "P": 1, "A": 2, "S": 2, "R": 2, "M": 2, "X": 0 }, "spans": { "F": 1, "G": 1, "P": 1, "A": 2, "S": 2, "R": 2, "M": 2, "X": 0 }, "primary": { "F": null, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "grayswan", "dimension": "F", "value": 1, "anchor": 1, "signals": [ "grayswan.03", "grayswan.06" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#667", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 1, "b": [ "grayswan.03" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "grayswan.03" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "grayswan.03" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "grayswan.03" ], "h": false, "c": false, "u": false, "x": [ "grayswan.06" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "grayswan.03", "grayswan.06" ] } }, "rationale_derived": "Capped at 1 by grayswan.03 (F.5): $40M Series A in May 2026 at a $200M valuation; revenue from every major lab.. Floors up to 1 from grayswan.06 do not exceed the cap.", "rationale_leads": "Capped at 1 by grayswan.03 (F.5): $40M Series A in May 2026 at a $200M valuation; revenue from every major lab.. Floors up to 1 from grayswan.06 do not exceed the cap." }, "G": { "evaluator": "grayswan", "dimension": "G", "value": 1, "anchor": 1, "signals": [ "grayswan.05", "grayswan.11", "grayswan.14" ], "assessed": "2026-09-15", "assessor": "yohei/claude v1.0", "graph_id": "assessment#668", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 1, "b": [ "grayswan.05" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "grayswan.05" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "grayswan.05" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "grayswan.05" ], "h": false, "c": false, "u": false, "x": [ "grayswan.14" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "grayswan.05", "grayswan.11", "grayswan.14" ] } }, "rationale_derived": "Capped at 1 by grayswan.05 (G.2): Venture-backed for-profit; no published COI policy found..", "rationale_leads": "Capped at 1 by grayswan.05 (G.2): Venture-backed for-profit; no published COI policy found.." }, "P": { "evaluator": "grayswan", "dimension": "P", "value": 1, "anchor": 1, "signals": [ "grayswan.01", "grayswan.02", "grayswan.12" ], "assessed": "2026-09-15", "assessor": "yohei/claude v1.0", "open_questions": [ "Request from Gray Swan a copy or description of the recusal arrangement governing the chief scientist's role between Gray Swan and OpenAI's board and Safety and Security Committee (date adopted, scope, who administers it), as reported by Forbes on 29 October 2024. Per RULES 12 this goes to the person before publication.", "Request from Gray Swan (or from the funds) a statement of whether Wing, Madrona, Obvious Ventures, Snowflake Ventures, Hudson River Trading, Samsung Next or Magarac Venture Partners hold equity in a frontier developer. The public record shows Snowflake's Anthropic tie as a $200M commercial partnership (3 Dec 2025), not equity." ], "graph_id": "assessment#669", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 1, "b": [ "grayswan.01", "grayswan.12" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "grayswan.01", "grayswan.12" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "grayswan.01", "grayswan.12" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "grayswan.01", "grayswan.12" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "grayswan.01", "grayswan.02", "grayswan.12" ] } }, "rationale_derived": "Capped at 1 by grayswan.01 (P.1): Co-founder and chief scientist chairs OpenAI's Safety and Security Committee and sits on its nonprofit board.; capped at 1 by grayswan.12 (P.1): Chief scientist appointed to OpenAI's board in 2024 and chairs its safety and security committee; also a 2025 Schmidt Sciences AI safety funding recipient.. Floors up to 1 from grayswan.02 do not exceed the cap.", "rationale_leads": "Capped at 1 by grayswan.01 (P.1): Co-founder and chief scientist chairs OpenAI's Safety and Security Committee and sits on its nonprofit board.; capped at 1 by grayswan.12 (P.1): Chief scientist appointed to OpenAI's board in 2024 and chairs its safety and security committee; also a 2025 Schmidt Sciences AI safety funding recipient.. Floors up to 1 from grayswan.02 do not exceed the cap." }, "A": { "evaluator": "grayswan", "dimension": "A", "value": 2, "anchor": 2, "signals": [ "grayswan.07" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "lab-controlled", "graph_id": "assessment#670", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "grayswan.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "grayswan.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "grayswan.07" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "grayswan.07" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "grayswan.07" ] } }, "rationale_derived": "Floored at 2 by grayswan.07 (A.1): Embedded in pre-release safety evaluation processes; cited in 11 system cards.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by grayswan.07 (A.1): Embedded in pre-release safety evaluation processes; cited in 11 system cards.. No admissible signal caps it." }, "S": { "evaluator": "grayswan", "dimension": "S", "value": 2, "anchor": 2, "signals": [ "grayswan.10" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#671", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "grayswan.10" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "grayswan.10" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "grayswan.10" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "grayswan.10" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "grayswan.10" ] } }, "rationale_derived": "Capped at 2 by grayswan.10 (S.2): Scope set by lab engagements..", "rationale_leads": "Capped at 2 by grayswan.10 (S.2): Scope set by lab engagements.." }, "R": { "evaluator": "grayswan", "dimension": "R", "value": 2, "anchor": 2, "signals": [ "grayswan.09" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "mechanism": "lab-controlled", "graph_id": "assessment#672", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "grayswan.09" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "grayswan.09" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "grayswan.09" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "grayswan.09" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "grayswan.09" ] } }, "rationale_derived": "Capped at 2 by grayswan.09 (R.3): Findings appear as lab-summarized system-card citations rather than independent reports..", "rationale_leads": "Capped at 2 by grayswan.09 (R.3): Findings appear as lab-summarized system-card citations rather than independent reports.." }, "M": { "evaluator": "grayswan", "dimension": "M", "value": 2, "anchor": 2, "signals": [ "grayswan.08" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#673", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "grayswan.08" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "grayswan.08" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "grayswan.08" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "grayswan.08" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "grayswan.08" ] } }, "rationale_derived": "Floored at 2 by grayswan.08 (M.2): Founders wrote foundational jailbreak papers; Arena results partly public.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by grayswan.08 (M.2): Founders wrote foundational jailbreak papers; Arena results partly public.. No admissible signal caps it." }, "X": { "evaluator": "grayswan", "dimension": "X", "value": 0, "anchor": 0, "signals": [ "grayswan.04", "grayswan.15" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#674", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 0, "b": [ "grayswan.04" ], "h": true, "c": false, "u": false, "x": [] }, "standard": { "v": 0, "b": [ "grayswan.04" ], "h": true, "c": false, "u": false, "x": [] }, "against_interest": { "v": 0, "b": [ "grayswan.04" ], "h": true, "c": false, "u": false, "x": [] }, "spans": { "v": 0, "b": [ "grayswan.04" ], "h": false, "c": false, "u": false, "x": [ "grayswan.15" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "grayswan.04", "grayswan.15" ] } }, "rationale_derived": "Capped at 0 by grayswan.04 (X.1): Sells Shade and Cygnal, the fixes, to the same labs whose models it evaluates..", "rationale_leads": "Capped at 0 by grayswan.04 (X.1): Sells Shade and Cygnal, the fixes, to the same labs whose models it evaluates.." } }, "signals": [ { "id": "grayswan.01", "evaluator": "grayswan", "dimension": "P", "direction": "against", "claim": "Co-founder and chief scientist chairs OpenAI's Safety and Security Committee and sits on its nonprofit board.", "sources": [ "openai-kolter-board", "kolter-bio" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "quote": "he chairs the Safety and Security Committee", "bound": { "cap": 1 }, "rule": "P.1", "bound_note": "Cap depends on the recusal fact carried by grayswan.02.", "quote_source": "kolter-bio", "graph_id": "signal#364", "source_graph_ids": [ "source#143", "source#102" ] }, { "id": "grayswan.02", "evaluator": "grayswan", "dimension": "P", "direction": "for", "claim": "The chief scientist recuses himself from Gray Swan's dealings with OpenAI.", "sources": [ "forbes-grayswan-2024", "grayswan-forbesau-2024" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "quote": "he has recused himself from interactions between the two companies", "bound": { "floor": 1 }, "rule": "P.1", "quote_source": "grayswan-forbesau-2024", "graph_id": "signal#365", "source_graph_ids": [ "source#84", "source#90" ] }, { "id": "grayswan.03", "evaluator": "grayswan", "dimension": "F", "direction": "against", "claim": "$40M Series A in May 2026 at a $200M valuation; revenue from every major lab.", "sources": [ "grayswan-seriesa", "forbes-grayswan-2026" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 1 }, "rule": "F.5", "quote": "Gray Swan, The AI Security Company Trusted by Every Major Frontier Lab, Raises $40M Series A", "quote_source": "grayswan-seriesa", "graph_id": "signal#366", "source_graph_ids": [ "source#92", "source#85" ] }, { "id": "grayswan.04", "evaluator": "grayswan", "dimension": "X", "direction": "against", "claim": "Sells Shade and Cygnal, the fixes, to the same labs whose models it evaluates.", "sources": [ "grayswan-seriesa", "madrona-grayswan" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "quote": "Cygnal for real-time AI protection, Shade for continuous adversarial testing", "bound": { "cap": 0 }, "rule": "X.1", "quote_source": "grayswan-seriesa", "graph_id": "signal#367", "source_graph_ids": [ "source#92", "source#107" ] }, { "id": "grayswan.05", "evaluator": "grayswan", "dimension": "G", "direction": "against", "claim": "Venture-backed for-profit; no published COI policy found.", "sources": [ "grayswan-seriesa" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 1 }, "rule": "G.2", "quote": "today announced a $40 million Series A round co-led by Wing Venture Capital and Madrona", "quote_source": "grayswan-seriesa", "graph_id": "signal#368", "source_graph_ids": [ "source#92" ] }, { "id": "grayswan.06", "evaluator": "grayswan", "dimension": "F", "direction": "for", "claim": "Breadth across OpenAI, Anthropic, Google DeepMind, Meta, xAI and ByteDance, so no single lab dominates.", "sources": [ "forbes-grayswan-2026" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 1 }, "rule": "F.5 (anchor text)", "graph_id": "signal#369", "source_graph_ids": [ "source#85" ] }, { "id": "grayswan.07", "evaluator": "grayswan", "dimension": "A", "direction": "for", "claim": "Embedded in pre-release safety evaluation processes; cited in 11 system cards.", "sources": [ "grayswan-seriesa", "madrona-grayswan" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "A.1", "quote": "Gray Swan’s benchmarks are embedded into the safety evaluation processes", "quote_source": "grayswan-seriesa", "graph_id": "signal#370", "source_graph_ids": [ "source#92", "source#107" ] }, { "id": "grayswan.08", "evaluator": "grayswan", "dimension": "M", "direction": "for", "claim": "Founders wrote foundational jailbreak papers; Arena results partly public.", "sources": [ "forbes-grayswan-2026", "grayswan-seriesa" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "M.2", "quote": "Research Published papers and findings from our research team", "quote_source": "grayswan-seriesa", "graph_id": "signal#371", "source_graph_ids": [ "source#85", "source#92" ] }, { "id": "grayswan.09", "evaluator": "grayswan", "dimension": "R", "direction": "against", "claim": "Findings appear as lab-summarized system-card citations rather than independent reports.", "sources": [ "grayswan-seriesa" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "R.3", "quote": "Gray Swan is a leading security evaluator cited in 11 recent frontier model system cards", "quote_source": "grayswan-seriesa", "graph_id": "signal#372", "source_graph_ids": [ "source#92" ] }, { "id": "grayswan.10", "evaluator": "grayswan", "dimension": "S", "direction": "against", "claim": "Scope set by lab engagements.", "sources": [ "grayswan-seriesa" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "S.2", "quote": "AI Red-Teaming Dedicated adversarial assessment, scoped to your deployment", "quote_source": "grayswan-seriesa", "graph_id": "signal#373", "source_graph_ids": [ "source#92" ] }, { "id": "grayswan.11", "evaluator": "grayswan", "dimension": "G", "direction": "against", "claim": "Series A co-led by Wing and Madrona with Obvious Ventures, Snowflake Ventures, Hudson River Trading, Samsung Next and Magarac; the co-lead's own post notes the chief scientist serves on OpenAI's board.", "sources": [ "madrona-grayswan", "dealroom-grayswan" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2026-05-28", "quote": "serves on the board of OpenAI", "bound": { "cap": 2 }, "rule": "G.2 (anchor text); G.3 not triggered", "quote_source": "madrona-grayswan", "graph_id": "signal#374", "source_graph_ids": [ "source#107", "source#58" ] }, { "id": "grayswan.12", "evaluator": "grayswan", "dimension": "P", "direction": "against", "claim": "Chief scientist appointed to OpenAI's board in 2024 and chairs its safety and security committee; also a 2025 Schmidt Sciences AI safety funding recipient.", "sources": [ "wikipedia-kolter" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2026-09-15", "quote": "became chair of its safety and security committee", "bound": { "cap": 1 }, "rule": "P.1", "quote_source": "wikipedia-kolter", "graph_id": "signal#375", "source_graph_ids": [ "source#200" ] }, { "id": "grayswan.14", "evaluator": "grayswan", "dimension": "G", "direction": "for", "claim": "Snowflake, a Series A co-investor, announced a commercial agreement with Anthropic rather than an equity stake, so the co-investor is not a lab investor under F.4 or G.3.", "sources": [ "grayswan-snowflake-anthropic" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": null, "bound_note": "Informational: closes the F.4/G.3 question for one co-investor; no floor follows from the absence of a tie (RULES 0).", "rule": "G.3", "graph_id": "signal#376", "source_graph_ids": [ "source#93" ] }, { "id": "grayswan.15", "evaluator": "grayswan", "dimension": "X", "direction": "against", "claim": "A podcast interview reports Shade, Gray Swan's defense product, in use at Anthropic, a lab whose models Gray Swan red-teams.", "sources": [ "grayswan-latentspace-2026" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 0 }, "rule": "X.1", "note": "Lead for X.1 (a documented lab customer for the fix); the cap is already set by grayswan.04.", "graph_id": "signal#377", "source_graph_ids": [ "source#91" ] } ], "scores": { "lab": 39, "regulator": 41, "public": 34, "equal": 34 }, "scores_by_policy": { "leads": { "lab": { "score": 39, "band": "disqualifying", "coverage": 8, "range": [ 39, 39 ] }, "regulator": { "score": 41, "band": "disqualifying", "coverage": 8, "range": [ 41, 41 ] }, "public": { "score": 34, "band": "disqualifying", "coverage": 8, "range": [ 34, 34 ] }, "equal": { "score": 34, "band": "disqualifying", "coverage": 8, "range": [ 34, 34 ] } }, "standard": { "lab": { "score": 39, "band": "disqualifying", "coverage": 8, "range": [ 39, 39 ] }, "regulator": { "score": 41, "band": "disqualifying", "coverage": 8, "range": [ 41, 41 ] }, "public": { "score": 34, "band": "disqualifying", "coverage": 8, "range": [ 34, 34 ] }, "equal": { "score": 34, "band": "disqualifying", "coverage": 8, "range": [ 34, 34 ] } }, "against_interest": { "lab": { "score": 39, "band": "disqualifying", "coverage": 8, "range": [ 39, 39 ] }, "regulator": { "score": 41, "band": "disqualifying", "coverage": 8, "range": [ 41, 41 ] }, "public": { "score": 34, "band": "disqualifying", "coverage": 8, "range": [ 34, 34 ] }, "equal": { "score": 34, "band": "disqualifying", "coverage": 8, "range": [ 34, 34 ] } }, "spans": { "lab": { "score": 39, "band": "disqualifying", "coverage": 8, "range": [ 39, 39 ] }, "regulator": { "score": 41, "band": "disqualifying", "coverage": 8, "range": [ 41, 41 ] }, "public": { "score": 34, "band": "disqualifying", "coverage": 8, "range": [ 34, 34 ] }, "equal": { "score": 34, "band": "disqualifying", "coverage": 8, "range": [ 34, 34 ] } }, "primary": { "lab": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "regulator": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "public": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "equal": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] } } }, "band": "disqualifying", "coverage": 8, "floor": "X", "what_moves": { "lab": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 43, "band": "disqualifying", "base": 39 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 41, "band": "disqualifying", "base": 39 }, { "dimension": "P", "from_value": 1, "to_value": 2, "score": 41, "band": "disqualifying", "base": 39 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 44, "band": "disqualifying", "base": 39 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 43, "band": "disqualifying", "base": 39 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 42, "band": "disqualifying", "base": 39 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 41, "band": "disqualifying", "base": 39 }, { "dimension": "X", "from_value": 0, "to_value": 1, "score": 40, "band": "conditional", "base": 39 } ], "regulator": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 45, "band": "disqualifying", "base": 41 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 44, "band": "disqualifying", "base": 41 }, { "dimension": "P", "from_value": 1, "to_value": 2, "score": 44, "band": "disqualifying", "base": 41 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 46, "band": "disqualifying", "base": 41 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 46, "band": "disqualifying", "base": 41 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 45, "band": "disqualifying", "base": 41 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 44, "band": "disqualifying", "base": 41 }, { "dimension": "X", "from_value": 0, "to_value": 1, "score": 41, "band": "conditional", "base": 41 } ], "public": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 40, "band": "disqualifying", "base": 34 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 38, "band": "disqualifying", "base": 34 }, { "dimension": "P", "from_value": 1, "to_value": 2, "score": 38, "band": "disqualifying", "base": 34 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 35, "band": "disqualifying", "base": 34 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 36, "band": "disqualifying", "base": 34 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 39, "band": "disqualifying", "base": 34 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 35, "band": "disqualifying", "base": 34 }, { "dimension": "X", "from_value": 0, "to_value": 1, "score": 35, "band": "conditional", "base": 34 } ], "equal": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 38, "band": "disqualifying", "base": 34 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 38, "band": "disqualifying", "base": 34 }, { "dimension": "P", "from_value": 1, "to_value": 2, "score": 38, "band": "disqualifying", "base": 34 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 38, "band": "disqualifying", "base": 34 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 38, "band": "disqualifying", "base": 34 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 38, "band": "disqualifying", "base": 34 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 38, "band": "disqualifying", "base": 34 }, { "dimension": "X", "from_value": 0, "to_value": 1, "score": 38, "band": "conditional", "base": 34 } ] } }, { "id": "hal", "name": "Holistic Agent Leaderboard (Princeton)", "type": "academic", "hq": "Princeton, US", "domains": [ "benchmarks", "autonomy" ], "confidence": "med", "summary": "Academic agent leaderboard with fully open methodology; AEF founding member.", "what_would_move_the_score": "Nothing on independence; access is the gap.", "role": "benchmark", "dissent": { "lower": "Access is already 0 and cannot go lower, so the weakest movable dimension is Personnel, which should be 1. A co-author is listed at xAI, staff of Anthropic, Google DeepMind and UK AISI are acknowledged on the page, OpenAI and Google supply the API credits, and no recusal or contributor conflict rule exists; lab-affiliated authors on a leaderboard that grades their employers is anchor 1's informal arrangement.", "higher": "Personnel should be 3. Every co-author's affiliation is disclosed on the page, the project sits inside Princeton, whose faculty are bound by a university conflict-of-interest policy, no HAL leader holds a lab role, and the leaderboard's scoring is open code anyone can re-run. Disclosed ties under an institutional policy is anchor 3, not anchor 2." }, "list_group": "referee", "graph_id": "evaluator#214", "values": { "F": 3, "G": 2, "P": 2, "A": 0, "S": 3, "R": 3, "M": 3, "X": 3 }, "values_by_policy": { "leads": { "F": 3, "G": 2, "P": 2, "A": 0, "S": 3, "R": 3, "M": 3, "X": 3 }, "standard": { "F": 3, "G": 2, "P": 2, "A": 0, "S": 3, "R": 3, "M": 3, "X": 3 }, "against_interest": { "F": 3, "G": 2, "P": 2, "A": 0, "S": null, "R": null, "M": null, "X": null }, "spans": { "F": 3, "G": 2, "P": 2, "A": 0, "S": 3, "R": 3, "M": 3, "X": 3 }, "primary": { "F": null, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "hal", "dimension": "F", "value": 3, "anchor": 3, "signals": [ "hal.01", "hal.09", "hal.10" ], "rationale": "Anchor 3 pending a bounded negative (university sponsorship disclosures or a funder index search).", "assessed": "2026-09-14", "assessor": "yohei/claude v0.4", "open_questions": [ "Request to the HAL team: the amount or approximate value of the OpenAI and Google API credits received and whether any evaluation depends on them, plus a funder list with amounts so a bounded negative on lab money can be recorded." ], "graph_id": "assessment#675", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "hal.09", "hal.10" ], "h": true, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "hal.09", "hal.10" ], "h": true, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "hal.09", "hal.10" ], "h": false, "c": false, "u": false, "x": [ "hal.01" ] }, "spans": { "v": 3, "b": [ "hal.09", "hal.10" ], "h": false, "c": false, "u": false, "x": [ "hal.01" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hal.01", "hal.09", "hal.10" ] } }, "rationale_derived": "Capped at 3 by hal.09 (F.11): University-hosted leaderboard; no lab money found, and no bounded negative on file (no filing or index searched yet).; capped at 3 by hal.10 (F.6): HAL is funded by Coefficient Giving, whose principal is an Anthropic investor, and receives API credits from OpenAI and Google to evaluate their models.. Floors up to 3 from hal.01 do not exceed the cap.", "rationale_leads": "Capped at 3 by hal.09 (F.11): University-hosted leaderboard; no lab money found, and no bounded negative on file (no filing or index searched yet).; capped at 3 by hal.10 (F.6): HAL is funded by Coefficient Giving, whose principal is an Anthropic investor, and receives API credits from OpenAI and Google to evaluate their models.. Floors up to 3 from hal.01 do not exceed the cap." }, "G": { "evaluator": "hal", "dimension": "G", "value": 2, "anchor": 2, "signals": [ "hal.05" ], "rationale": "Anchor 3: university-hosted academic project and AEF member (hal.05). No tier-1 source for an independent board or external review, so 4 is unearned under the tier rule (C10).", "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#676", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "hal.05" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "hal.05" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "hal.05" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "hal.05" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hal.05" ] } }, "rationale_derived": "Floored at 2 by hal.05 (G.1): University-hosted; AEF member.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by hal.05 (G.1): University-hosted; AEF member.. No admissible signal caps it." }, "P": { "evaluator": "hal", "dimension": "P", "value": 2, "anchor": 2, "signals": [ "hal.06", "hal.11" ], "rationale": "Anchor 3 (provisional): academic staff (hal.06); no lab-tie disclosures or cooling-off periods are documented at any tier, so 4 is unearned under the tier rule (C10). Provisional readings are open to public correction through the contribution path.", "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "open_questions": [ "Request to the HAL team at Princeton: the contributor conflict-of-interest or recusal rule that applies to the leaderboard, if any, and the date on which the xAI-affiliated co-author's affiliation began relative to the runs they contributed." ], "graph_id": "assessment#677", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "hal.11" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "hal.11" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "hal.11" ], "h": false, "c": false, "u": false, "x": [ "hal.06" ] }, "spans": { "v": 2, "b": [ "hal.11" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hal.06", "hal.11" ] } }, "rationale_derived": "Capped at 2 by hal.11 (P.4): HAL's author list includes a co-author affiliated with xAI, and its acknowledgments name staff at Anthropic, Google DeepMind and UK AISI; no contributor recusal or cooling-off rule is published.. Floors up to 2 from hal.06 do not exceed the cap.", "rationale_leads": "Capped at 2 by hal.11 (P.4): HAL's author list includes a co-author affiliated with xAI, and its acknowledgments name staff at Anthropic, Google DeepMind and UK AISI; no contributor recusal or cooling-off rule is published.. Floors up to 2 from hal.06 do not exceed the cap." }, "A": { "evaluator": "hal", "dimension": "A", "value": 0, "anchor": 0, "signals": [ "hal.04" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "lab-controlled", "graph_id": "assessment#678", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 0, "b": [ "hal.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 0, "b": [ "hal.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 0, "b": [ "hal.04" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 0, "b": [ "hal.04" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hal.04" ] } }, "rationale_derived": "Capped at 0 by hal.04 (A.6): Public API access only..", "rationale_leads": "Capped at 0 by hal.04 (A.6): Public API access only.." }, "S": { "evaluator": "hal", "dimension": "S", "value": 3, "anchor": 3, "signals": [ "hal.07" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#679", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "hal.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "hal.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hal.07" ] }, "spans": { "v": 3, "b": [ "hal.07" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hal.07" ] } }, "rationale_derived": "Floored at 3 by hal.07 (S.5): Own benchmark design.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by hal.07 (S.5): Own benchmark design.. No admissible signal caps it." }, "R": { "evaluator": "hal", "dimension": "R", "value": 3, "anchor": 3, "signals": [ "hal.03" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "self-imposed", "graph_id": "assessment#680", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "hal.03" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "hal.03" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hal.03" ] }, "spans": { "v": 3, "b": [ "hal.03" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hal.03" ] } }, "rationale_derived": "Floored at 3 by hal.03 (R.5): Results published without review.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by hal.03 (R.5): Results published without review.. No admissible signal caps it." }, "M": { "evaluator": "hal", "dimension": "M", "value": 3, "anchor": 3, "signals": [ "hal.02" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#681", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "hal.02" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "hal.02" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hal.02" ] }, "spans": { "v": 3, "b": [ "hal.02" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hal.02" ] } }, "rationale_derived": "Floored at 3 by hal.02 (M.1): Open code and logs.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by hal.02 (M.1): Open code and logs.. No admissible signal caps it." }, "X": { "evaluator": "hal", "dimension": "X", "value": 3, "anchor": 3, "signals": [ "hal.08" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "evidence_limited": true, "graph_id": "assessment#682", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "hal.08" ], "h": true, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "hal.08" ], "h": true, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hal.08" ] }, "spans": { "v": 3, "b": [ "hal.08" ], "h": true, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hal.08" ] } }, "rationale_derived": "Floored at 3 by hal.08 (X.7): No products.. No admissible signal caps it. Held: C16: a 4 needs a tier-1/2 source or two independent sources, one not self-published.", "rationale_leads": "Floored at 3 by hal.08 (X.7): No products.. No admissible signal caps it. Held: C16: a 4 needs a tier-1/2 source or two independent sources, one not self-published." } }, "signals": [ { "id": "hal.01", "evaluator": "hal", "dimension": "F", "direction": "for", "claim": "No lab money; academic.", "sources": [ "hal" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 4 }, "rule": "F.11", "graph_id": "signal#378", "source_graph_ids": [ "source#94" ] }, { "id": "hal.02", "evaluator": "hal", "dimension": "M", "direction": "for", "claim": "Open code and logs.", "sources": [ "hal" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "M.1", "quote": "HAL is an open-source project and we welcome contributions from the community", "quote_source": "hal", "graph_id": "signal#379", "source_graph_ids": [ "source#94" ] }, { "id": "hal.03", "evaluator": "hal", "dimension": "R", "direction": "for", "claim": "Results published without review.", "sources": [ "hal" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "R.5", "quote": "The standardized, cost-aware, and third-party leaderboard for evaluating agents", "quote_source": "hal", "graph_id": "signal#380", "source_graph_ids": [ "source#94" ] }, { "id": "hal.04", "evaluator": "hal", "dimension": "A", "direction": "against", "claim": "Public API access only.", "sources": [ "hal" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 0 }, "rule": "A.6", "quote": "We are grateful to OpenAI and Google for providing API credits to evaluate their models", "quote_source": "hal", "graph_id": "signal#381", "source_graph_ids": [ "source#94" ] }, { "id": "hal.05", "evaluator": "hal", "dimension": "G", "direction": "for", "claim": "University-hosted; AEF member.", "sources": [ "hal", "aef-launch" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "G.1", "quote": "By the SAgE team at Princeton University", "quote_source": "hal", "graph_id": "signal#382", "source_graph_ids": [ "source#94", "source#10" ] }, { "id": "hal.06", "evaluator": "hal", "dimension": "P", "direction": "for", "claim": "Academic staff.", "sources": [ "hal" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "P.6", "quote": "Sayash Kapoor Princeton University Benedikt Stroebl Princeton University", "quote_source": "hal", "graph_id": "signal#383", "source_graph_ids": [ "source#94" ] }, { "id": "hal.07", "evaluator": "hal", "dimension": "S", "direction": "for", "claim": "Own benchmark design.", "sources": [ "hal" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "S.5", "quote": "We have paused updating HAL leaderboard with new models and are currently focusing on measuring reliability in AI agents", "quote_source": "hal", "graph_id": "signal#384", "source_graph_ids": [ "source#94" ] }, { "id": "hal.08", "evaluator": "hal", "dimension": "X", "direction": "for", "claim": "No products.", "sources": [ "hal" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 4 }, "rule": "X.7", "quote": "HAL is an open-source project", "quote_source": "hal", "graph_id": "signal#385", "source_graph_ids": [ "source#94" ] }, { "id": "hal.09", "evaluator": "hal", "dimension": "F", "direction": "against", "claim": "University-hosted leaderboard; no lab money found, and no bounded negative on file (no filing or index searched yet).", "sources": [ "hal" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 3 }, "rule": "F.11", "quote": "HAL is funded by Coefficient Giving , Schmidt Sciences , the Princeton AI Lab", "quote_source": "hal", "graph_id": "signal#386", "source_graph_ids": [ "source#94" ] }, { "id": "hal.10", "evaluator": "hal", "dimension": "F", "direction": "against", "claim": "HAL is funded by Coefficient Giving, whose principal is an Anthropic investor, and receives API credits from OpenAI and Google to evaluate their models.", "sources": [ "hal", "anthropic-series-a" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "F.6", "quote": "We are grateful to OpenAI and Google for providing API credits to evaluate their models", "quote_source": "hal", "note": "F.6 caps Coefficient money at 3 ('HAL is funded by Coefficient Giving , Schmidt Sciences'; Moskovitz participated in Anthropic's Series A). The lab API credits are undisclosed in-kind with no dependency statement, a gap in F.2; by anchor text they exclude 'no lab money'.", "graph_id": "signal#387", "source_graph_ids": [ "source#94", "source#28" ] }, { "id": "hal.11", "evaluator": "hal", "dimension": "P", "direction": "against", "claim": "HAL's author list includes a co-author affiliated with xAI, and its acknowledgments name staff at Anthropic, Google DeepMind and UK AISI; no contributor recusal or cooling-off rule is published.", "sources": [ "hal" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 2 }, "rule": "P.4", "quote": "Yifei Zhou xAI", "quote_source": "hal", "note": "A lab employee among the project's contributors with no recusal rule caps at 2 (P.4 by analogy; P.3 if the affiliation followed the contribution). Public role only; the open question is a document request.", "graph_id": "signal#388", "source_graph_ids": [ "source#94" ] } ], "scores": { "lab": 56, "regulator": 55, "public": 64, "equal": 59 }, "scores_by_policy": { "leads": { "lab": { "score": 56, "band": "disqualifying", "coverage": 8, "range": [ 56, 56 ] }, "regulator": { "score": 55, "band": "disqualifying", "coverage": 8, "range": [ 55, 55 ] }, "public": { "score": 64, "band": "disqualifying", "coverage": 8, "range": [ 64, 64 ] }, "equal": { "score": 59, "band": "disqualifying", "coverage": 8, "range": [ 59, 59 ] } }, "standard": { "lab": { "score": 56, "band": "disqualifying", "coverage": 8, "range": [ 56, 56 ] }, "regulator": { "score": 55, "band": "disqualifying", "coverage": 8, "range": [ 55, 55 ] }, "public": { "score": 64, "band": "disqualifying", "coverage": 8, "range": [ 64, 64 ] }, "equal": { "score": 59, "band": "disqualifying", "coverage": 8, "range": [ 59, 59 ] } }, "against_interest": { "lab": { "score": 40, "band": "disqualifying", "coverage": 4, "range": [ 23, 67 ] }, "regulator": { "score": 39, "band": "disqualifying", "coverage": 4, "range": [ 21, 66 ] }, "public": { "score": 56, "band": "disqualifying", "coverage": 4, "range": [ 34, 74 ] }, "equal": { "score": 44, "band": "disqualifying", "coverage": 4, "range": [ 22, 72 ] } }, "spans": { "lab": { "score": 56, "band": "disqualifying", "coverage": 8, "range": [ 56, 56 ] }, "regulator": { "score": 55, "band": "disqualifying", "coverage": 8, "range": [ 55, 55 ] }, "public": { "score": 64, "band": "disqualifying", "coverage": 8, "range": [ 64, 64 ] }, "equal": { "score": 59, "band": "disqualifying", "coverage": 8, "range": [ 59, 59 ] } }, "primary": { "lab": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "regulator": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "public": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "equal": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] } } }, "band": "disqualifying", "coverage": 8, "floor": "A", "what_moves": { "lab": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 60, "band": "disqualifying", "base": 56 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 57, "band": "disqualifying", "base": 56 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 58, "band": "disqualifying", "base": 56 }, { "dimension": "A", "from_value": 0, "to_value": 1, "score": 61, "band": "conditional", "base": 56 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 60, "band": "disqualifying", "base": 56 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 59, "band": "disqualifying", "base": 56 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 58, "band": "disqualifying", "base": 56 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 57, "band": "disqualifying", "base": 56 } ], "regulator": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 59, "band": "disqualifying", "base": 55 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 57, "band": "disqualifying", "base": 55 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 57, "band": "disqualifying", "base": 55 }, { "dimension": "A", "from_value": 0, "to_value": 1, "score": 60, "band": "conditional", "base": 55 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 60, "band": "disqualifying", "base": 55 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 59, "band": "disqualifying", "base": 55 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 57, "band": "disqualifying", "base": 55 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 55, "band": "disqualifying", "base": 55 } ], "public": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 70, "band": "disqualifying", "base": 64 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 68, "band": "disqualifying", "base": 64 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 68, "band": "disqualifying", "base": 64 }, { "dimension": "A", "from_value": 0, "to_value": 1, "score": 65, "band": "conditional", "base": 64 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 66, "band": "disqualifying", "base": 64 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 69, "band": "disqualifying", "base": 64 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 65, "band": "disqualifying", "base": 64 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 65, "band": "disqualifying", "base": 64 } ], "equal": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 63, "band": "disqualifying", "base": 59 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 63, "band": "disqualifying", "base": 59 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 63, "band": "disqualifying", "base": 59 }, { "dimension": "A", "from_value": 0, "to_value": 1, "score": 63, "band": "conditional", "base": 59 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 63, "band": "disqualifying", "base": 59 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 63, "band": "disqualifying", "base": 59 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 63, "band": "disqualifying", "base": 59 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 63, "band": "disqualifying", "base": 59 } ] } }, { "id": "hfoai", "name": "Hugging Face Open Alignment Initiative", "type": "vc", "hq": "New York, US / Paris, FR", "domains": [ "assurance" ], "confidence": "low", "summary": "Announced 12 September 2026 with a request to join Anthropic's embedded-evaluator program; no evaluations yet.", "what_would_move_the_score": "A governance structure that insulates the initiative from Nvidia, and a first published evaluation.", "role": "expected-entrant", "status": "watchlist", "dissent": { "lower": "Role incompatibility (2 to 1). Hugging Face sells PRO, Team and Enterprise plans and paid inference to the same ecosystem it proposes to audit, and frontier developers distribute models through the Hub. If any evaluated lab is a paying Enterprise Hub or inference customer, anchor 1 (sells products or services to labs) applies rather than the 2 that holds only until a lab customer is documented.", "higher": "Access depth (0 to 1). Anthropic committed on 12 September to give third-party evaluators permanent, employee-level access, OpenAI said it would match, and Hugging Face asked to join within hours with a named lead. If admitted, the initiative's first engagement would begin above public-API access. The record shows a live programme and a pending request, not a refusal." }, "list_group": "referee", "graph_id": "evaluator#215", "values": { "F": 0, "G": 0, "P": 2, "A": 0, "S": 2, "R": 2, "M": 2, "X": 2 }, "values_by_policy": { "leads": { "F": 0, "G": 0, "P": 2, "A": 0, "S": 2, "R": 2, "M": 2, "X": 2 }, "standard": { "F": 0, "G": 0, "P": 2, "A": 0, "S": 2, "R": 2, "M": 2, "X": 2 }, "against_interest": { "F": 0, "G": 0, "P": 2, "A": 0, "S": 2, "R": 2, "M": 2, "X": 2 }, "spans": { "F": 0, "G": 0, "P": 2, "A": 0, "S": 2, "R": 2, "M": null, "X": 2 }, "primary": { "F": null, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "hfoai", "dimension": "F", "value": 0, "anchor": 0, "signals": [ "hfoai.03", "hfoai.09", "hfoai.10" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#683", "evidence_tier": "tier 2 (ledger)", "derived": { "leads": { "v": 0, "b": [ "hfoai.10" ], "h": true, "c": false, "u": false, "x": [] }, "standard": { "v": 0, "b": [ "hfoai.10" ], "h": true, "c": false, "u": false, "x": [] }, "against_interest": { "v": 0, "b": [ "hfoai.10" ], "h": true, "c": false, "u": false, "x": [] }, "spans": { "v": 0, "b": [ "hfoai.10" ], "h": true, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hfoai.03", "hfoai.09", "hfoai.10" ] } }, "rationale_derived": "Capped at 0 by hfoai.10 (F.1): Nvidia agreed to acquire Hugging Face for $12,930,300,000 on 3 September 2026..", "rationale_leads": "Capped at 0 by hfoai.10 (F.1): Nvidia agreed to acquire Hugging Face for $12,930,300,000 on 3 September 2026.." }, "G": { "evaluator": "hfoai", "dimension": "G", "value": 0, "anchor": 0, "signals": [ "hfoai.01" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#684", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 0, "b": [ "hfoai.01" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 0, "b": [ "hfoai.01" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 0, "b": [ "hfoai.01" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 0, "b": [ "hfoai.01" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hfoai.01" ] } }, "rationale_derived": "Capped at 0 by hfoai.01 (G.6): Nvidia is acquiring Hugging Face for about $12.9B and is reported to be in talks to anchor Anthropic's IPO with up to $10B; nobody involved has addressed the overlap..", "rationale_leads": "Capped at 0 by hfoai.01 (G.6): Nvidia is acquiring Hugging Face for about $12.9B and is reported to be in talks to anchor Anthropic's IPO with up to $10B; nobody involved has addressed the overlap.." }, "P": { "evaluator": "hfoai", "dimension": "P", "value": 2, "anchor": 2, "signals": [ "hfoai.07" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#685", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 2, "b": [ "hfoai.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "hfoai.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "hfoai.07" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "hfoai.07" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hfoai.07" ] } }, "rationale_derived": "Capped at 2 by hfoai.07 (P.4): Would be owned by a lab investor..", "rationale_leads": "Capped at 2 by hfoai.07 (P.4): Would be owned by a lab investor.." }, "A": { "evaluator": "hfoai", "dimension": "A", "value": 0, "anchor": 0, "signals": [ "hfoai.02" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "mechanism": "unknown", "graph_id": "assessment#686", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 0, "b": [ "hfoai.02" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 0, "b": [ "hfoai.02" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 0, "b": [ "hfoai.02" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 0, "b": [ "hfoai.02" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hfoai.02" ] } }, "rationale_derived": "Capped at 0 by hfoai.02 (A.6): No track record as an evaluator; no access yet..", "rationale_leads": "Capped at 0 by hfoai.02 (A.6): No track record as an evaluator; no access yet.." }, "S": { "evaluator": "hfoai", "dimension": "S", "value": 2, "anchor": 2, "signals": [ "hfoai.06" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#687", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 2, "b": [ "hfoai.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "hfoai.06" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "hfoai.06" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "hfoai.06" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hfoai.06" ] } }, "rationale_derived": "Capped at 2 by hfoai.06 (S.2 (anchor text)): Terms not announced..", "rationale_leads": "Capped at 2 by hfoai.06 (S.2 (anchor text)): Terms not announced.." }, "R": { "evaluator": "hfoai", "dimension": "R", "value": 2, "anchor": 2, "signals": [ "hfoai.04" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "self-imposed", "graph_id": "assessment#688", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 2, "b": [ "hfoai.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "hfoai.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "hfoai.04" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "hfoai.04" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hfoai.04" ] } }, "rationale_derived": "Floored at 2 by hfoai.04 (9): States findings would be published openly; open-source culture.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by hfoai.04 (9): States findings would be published openly; open-source culture.. No admissible signal caps it." }, "M": { "evaluator": "hfoai", "dimension": "M", "value": 2, "anchor": 2, "signals": [ "hfoai.05" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#689", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 2, "b": [ "hfoai.05" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "hfoai.05" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "hfoai.05" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hfoai.05" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hfoai.05" ] } }, "rationale_derived": "Floored at 2 by hfoai.05 (9): Public tooling.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by hfoai.05 (9): Public tooling.. No admissible signal caps it." }, "X": { "evaluator": "hfoai", "dimension": "X", "value": 2, "anchor": 2, "signals": [ "hfoai.08" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#690", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "hfoai.08" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "hfoai.08" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "hfoai.08" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "hfoai.08" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "hfoai.08" ] } }, "rationale_derived": "Capped at 2 by hfoai.08 (X.1 (second sentence)): Commercial platform..", "rationale_leads": "Capped at 2 by hfoai.08 (X.1 (second sentence)): Commercial platform.." } }, "signals": [ { "id": "hfoai.01", "evaluator": "hfoai", "dimension": "G", "direction": "against", "claim": "Nvidia is acquiring Hugging Face for about $12.9B and is reported to be in talks to anchor Anthropic's IPO with up to $10B; nobody involved has addressed the overlap.", "sources": [ "tnw-hf-nvidia" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "quote": "Nvidia confirmed on 3 September that it is buying Hugging Face for $12.93bn", "bound": { "cap": 0 }, "rule": "G.6", "quote_source": "tnw-hf-nvidia", "graph_id": "signal#389", "source_graph_ids": [ "source#187" ] }, { "id": "hfoai.02", "evaluator": "hfoai", "dimension": "A", "direction": "against", "claim": "No track record as an evaluator; no access yet.", "sources": [ "tnw-hf-nvidia", "techmeme-hf" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 0 }, "rule": "A.6", "quote": "Hugging Face has asked to become one of the outside auditors of the AI labs", "quote_source": "tnw-hf-nvidia", "graph_id": "signal#390", "source_graph_ids": [ "source#187", "source#179" ] }, { "id": "hfoai.03", "evaluator": "hfoai", "dimension": "F", "direction": "against", "claim": "Who pays, who selects, and on what terms is unresolved.", "sources": [ "tnw-hf-nvidia", "zvi-pace" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "F.10", "quote": "Nobody has said who pays the embedded evaluators", "quote_source": "tnw-hf-nvidia", "graph_id": "signal#391", "source_graph_ids": [ "source#187", "source#202" ] }, { "id": "hfoai.04", "evaluator": "hfoai", "dimension": "R", "direction": "for", "claim": "States findings would be published openly; open-source culture.", "sources": [ "techmeme-hf" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "9", "quote": "won't be solved behind the closed doors of a handful of frontier labs", "quote_source": "techmeme-hf", "graph_id": "signal#392", "source_graph_ids": [ "source#179" ] }, { "id": "hfoai.05", "evaluator": "hfoai", "dimension": "M", "direction": "for", "claim": "Public tooling.", "sources": [ "techmeme-hf" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "9", "graph_id": "signal#393", "source_graph_ids": [ "source#179" ] }, { "id": "hfoai.06", "evaluator": "hfoai", "dimension": "S", "direction": "against", "claim": "Terms not announced.", "sources": [ "tnw-hf-nvidia" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "S.2 (anchor text)", "quote": "Neither lab has said who gets in, on what terms, or who decides", "quote_source": "tnw-hf-nvidia", "graph_id": "signal#394", "source_graph_ids": [ "source#187" ] }, { "id": "hfoai.07", "evaluator": "hfoai", "dimension": "P", "direction": "against", "claim": "Would be owned by a lab investor.", "sources": [ "tnw-hf-nvidia" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "P.4", "quote": "Nvidia was in talks to put up to $10bn into Anthropic’s IPO as an anchor investor", "quote_source": "tnw-hf-nvidia", "graph_id": "signal#395", "source_graph_ids": [ "source#187" ] }, { "id": "hfoai.08", "evaluator": "hfoai", "dimension": "X", "direction": "against", "claim": "Commercial platform.", "sources": [ "tnw-hf-nvidia", "hfoai-hf-pricing" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "X.1 (second sentence)", "quote": "Instant setup for growing teams $20 /month per user", "quote_source": "hfoai-hf-pricing", "graph_id": "signal#396", "source_graph_ids": [ "source#187", "source#95" ] }, { "id": "hfoai.09", "evaluator": "hfoai", "dimension": "F", "direction": "against", "claim": "Series D (2023) investors include Google, Amazon and Nvidia; Nvidia's reported acquisition would make a lab investor the owner.", "sources": [ "evaluators-ledger", "tnw-hf-nvidia" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 0 }, "rule": "F.1", "quote": "Nvidia confirmed on 3 September that it is buying Hugging Face for $12.93bn", "quote_source": "tnw-hf-nvidia", "graph_id": "signal#397", "source_graph_ids": [ "source#81", "source#187" ] }, { "id": "hfoai.10", "evaluator": "hfoai", "dimension": "F", "direction": "against", "claim": "Nvidia agreed to acquire Hugging Face for $12,930,300,000 on 3 September 2026.", "sources": [ "fortune-hf-nvidia", "tnw-hf-nvidia" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2026-09-11", "quote": "Nvidia has agreed to acquire Hugging Face for $12,930,300,000", "bound": { "cap": 0 }, "rule": "F.1", "quote_source": "fortune-hf-nvidia", "graph_id": "signal#398", "source_graph_ids": [ "source#87", "source#187" ] } ], "scores": { "lab": 27, "regulator": 28, "public": 28, "equal": 31 }, "scores_by_policy": { "leads": { "lab": { "score": 27, "band": "disqualifying", "coverage": 8, "range": [ 27, 27 ] }, "regulator": { "score": 28, "band": "disqualifying", "coverage": 8, "range": [ 28, 28 ] }, "public": { "score": 28, "band": "disqualifying", "coverage": 8, "range": [ 28, 28 ] }, "equal": { "score": 31, "band": "disqualifying", "coverage": 8, "range": [ 31, 31 ] } }, "standard": { "lab": { "score": 27, "band": "disqualifying", "coverage": 8, "range": [ 27, 27 ] }, "regulator": { "score": 28, "band": "disqualifying", "coverage": 8, "range": [ 28, 28 ] }, "public": { "score": 28, "band": "disqualifying", "coverage": 8, "range": [ 28, 28 ] }, "equal": { "score": 31, "band": "disqualifying", "coverage": 8, "range": [ 31, 31 ] } }, "against_interest": { "lab": { "score": 27, "band": "disqualifying", "coverage": 8, "range": [ 27, 27 ] }, "regulator": { "score": 28, "band": "disqualifying", "coverage": 8, "range": [ 28, 28 ] }, "public": { "score": 28, "band": "disqualifying", "coverage": 8, "range": [ 28, 28 ] }, "equal": { "score": 31, "band": "disqualifying", "coverage": 8, "range": [ 31, 31 ] } }, "spans": { "lab": { "score": 25, "band": "disqualifying", "coverage": 7, "range": [ 23, 32 ] }, "regulator": { "score": 25, "band": "disqualifying", "coverage": 7, "range": [ 23, 33 ] }, "public": { "score": 26, "band": "disqualifying", "coverage": 7, "range": [ 25, 30 ] }, "equal": { "score": 29, "band": "disqualifying", "coverage": 7, "range": [ 25, 38 ] } }, "primary": { "lab": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "regulator": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "public": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "equal": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] } } }, "band": "disqualifying", "coverage": 8, "floor": "F", "what_moves": { "lab": [ { "dimension": "F", "from_value": 0, "to_value": 1, "score": 32, "band": "disqualifying", "base": 27 }, { "dimension": "G", "from_value": 0, "to_value": 1, "score": 29, "band": "disqualifying", "base": 27 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 30, "band": "disqualifying", "base": 27 }, { "dimension": "A", "from_value": 0, "to_value": 1, "score": 32, "band": "disqualifying", "base": 27 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 31, "band": "disqualifying", "base": 27 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 31, "band": "disqualifying", "base": 27 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 29, "band": "disqualifying", "base": 27 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 28, "band": "disqualifying", "base": 27 } ], "regulator": [ { "dimension": "F", "from_value": 0, "to_value": 1, "score": 31, "band": "disqualifying", "base": 28 }, { "dimension": "G", "from_value": 0, "to_value": 1, "score": 30, "band": "disqualifying", "base": 28 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 30, "band": "disqualifying", "base": 28 }, { "dimension": "A", "from_value": 0, "to_value": 1, "score": 33, "band": "disqualifying", "base": 28 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 33, "band": "disqualifying", "base": 28 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 31, "band": "disqualifying", "base": 28 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 30, "band": "disqualifying", "base": 28 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 28, "band": "disqualifying", "base": 28 } ], "public": [ { "dimension": "F", "from_value": 0, "to_value": 1, "score": 34, "band": "disqualifying", "base": 28 }, { "dimension": "G", "from_value": 0, "to_value": 1, "score": 31, "band": "disqualifying", "base": 28 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 31, "band": "disqualifying", "base": 28 }, { "dimension": "A", "from_value": 0, "to_value": 1, "score": 29, "band": "disqualifying", "base": 28 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 30, "band": "disqualifying", "base": 28 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 33, "band": "disqualifying", "base": 28 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 29, "band": "disqualifying", "base": 28 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 29, "band": "disqualifying", "base": 28 } ], "equal": [ { "dimension": "F", "from_value": 0, "to_value": 1, "score": 34, "band": "disqualifying", "base": 31 }, { "dimension": "G", "from_value": 0, "to_value": 1, "score": 34, "band": "disqualifying", "base": 31 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 34, "band": "disqualifying", "base": 31 }, { "dimension": "A", "from_value": 0, "to_value": 1, "score": 34, "band": "disqualifying", "base": 31 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 34, "band": "disqualifying", "base": 31 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 34, "band": "disqualifying", "base": 31 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 34, "band": "disqualifying", "base": 31 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 34, "band": "disqualifying", "base": 31 } ] } }, { "id": "humane", "name": "Humane Intelligence", "type": "nonprofit", "hq": "New York, US", "domains": [ "misuse", "jailbreak" ], "confidence": "med", "summary": "Public red-teaming exercises with NIST and IMDA; societal harms rather than catastrophic risk.", "what_would_move_the_score": "Pre-release access for the harms it covers.", "role": "referee", "dissent": { "lower": "Access should be 0. No pre-release access is documented anywhere: the public exercises, bias bounties and NIST and IMDA partnerships tested deployed models, no system card names Humane Intelligence as a pre-release tester, and no lab engagement is on record. The card shows 1 only because the anchor-0 claim lacks a quoted span (C15), not because anything above public access was ever granted.", "higher": "Access should be 2. Humane Intelligence co-ran the NIST ARIA pilot and the Singapore IMDA multilingual red-teaming exercise, in which participating developers supplied models under government programs; A.4 treats government testing agreements as 3. If those exercises gave structured access beyond the public API, even with safeguards on, the documented engagement is at least anchor 2." }, "list_group": "referee", "graph_id": "evaluator#216", "values": { "F": 3, "G": 2, "P": 2, "A": 1, "S": 3, "R": 3, "M": 2, "X": 2 }, "values_by_policy": { "leads": { "F": 3, "G": 2, "P": 2, "A": 1, "S": 3, "R": 3, "M": 2, "X": 2 }, "standard": { "F": 3, "G": 2, "P": 2, "A": 1, "S": 3, "R": 3, "M": 2, "X": 2 }, "against_interest": { "F": null, "G": null, "P": 2, "A": 1, "S": null, "R": null, "M": null, "X": 2 }, "spans": { "F": 3, "G": 2, "P": 2, "A": null, "S": 3, "R": null, "M": 2, "X": 2 }, "primary": { "F": null, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "humane", "dimension": "F", "value": 3, "anchor": 3, "signals": [ "humane.02" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#691", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "humane.02" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "humane.02" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "humane.02" ] }, "spans": { "v": 3, "b": [ "humane.02" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "humane.02" ] } }, "rationale_derived": "Floored at 3 by humane.02 (F.9): Government partners rather than lab clients.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by humane.02 (F.9): Government partners rather than lab clients.. No admissible signal caps it." }, "G": { "evaluator": "humane", "dimension": "G", "value": 2, "anchor": 2, "signals": [ "humane.04" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#692", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "humane.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "humane.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "humane.04" ] }, "spans": { "v": 2, "b": [ "humane.04" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "humane.04" ] } }, "rationale_derived": "Floored at 2 by humane.04 (G.1): Nonprofit.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by humane.04 (G.1): Nonprofit.. No admissible signal caps it." }, "P": { "evaluator": "humane", "dimension": "P", "value": 2, "anchor": 2, "signals": [ "humane.05", "humane.10" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "open_questions": [ "Request to Humane Intelligence: the board and advisory-group conflict-of-interest policy, and whether advisory-group members employed by a model developer recuse from evaluations involving that developer's models.", "Request to Humane Intelligence: whether any cooling-off rule applies to directors previously employed by a model developer, and the dates of the board president's OpenAI employment as stated in the published bio." ], "graph_id": "assessment#693", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "humane.10" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "humane.10" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "humane.10" ], "h": false, "c": false, "u": false, "x": [ "humane.05" ] }, "spans": { "v": 2, "b": [ "humane.10" ], "h": false, "c": false, "u": false, "x": [ "humane.05" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "humane.05", "humane.10" ] } }, "rationale_derived": "Capped at 2 by humane.10 (P.4): Humane Intelligence's advisory group includes a Senior Research Scientist at Meta, and its board president previously worked on OpenAI's Human Data team; no recusal or cooling-off rule is published.. Floors up to 2 from humane.05 do not exceed the cap.", "rationale_leads": "Capped at 2 by humane.10 (P.4): Humane Intelligence's advisory group includes a Senior Research Scientist at Meta, and its board president previously worked on OpenAI's Human Data team; no recusal or cooling-off rule is published.. Floors up to 2 from humane.05 do not exceed the cap." }, "A": { "evaluator": "humane", "dimension": "A", "value": 1, "anchor": 1, "signals": [ "humane.03" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "mechanism": "lab-controlled", "evidence_limited": true, "graph_id": "assessment#694", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 1, "b": [ "humane.03" ], "h": true, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "humane.03" ], "h": true, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "humane.03" ], "h": true, "c": false, "u": false, "x": [] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "humane.03" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "humane.03" ] } }, "rationale_derived": "Capped at 1 by humane.03 (A.6): Public-model access only.. Held: C15: a 0 needs a quoted span.", "rationale_leads": "Capped at 1 by humane.03 (A.6): Public-model access only.. Held: C15: a 0 needs a quoted span." }, "S": { "evaluator": "humane", "dimension": "S", "value": 3, "anchor": 3, "signals": [ "humane.06" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#695", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "humane.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "humane.06" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "humane.06" ] }, "spans": { "v": 3, "b": [ "humane.06" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "humane.06" ] } }, "rationale_derived": "Floored at 3 by humane.06 (S.5): Designs its own exercises.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by humane.06 (S.5): Designs its own exercises.. No admissible signal caps it." }, "R": { "evaluator": "humane", "dimension": "R", "value": 3, "anchor": 3, "signals": [ "humane.01" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "self-imposed", "graph_id": "assessment#696", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "humane.01" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "humane.01" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "humane.01" ] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "humane.01" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "humane.01" ] } }, "rationale_derived": "Floored at 3 by humane.01 (R.5): Publishes openly; public exercises.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by humane.01 (R.5): Publishes openly; public exercises.. No admissible signal caps it." }, "M": { "evaluator": "humane", "dimension": "M", "value": 2, "anchor": 2, "signals": [ "humane.07" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#697", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "humane.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "humane.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "humane.07" ] }, "spans": { "v": 2, "b": [ "humane.07" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "humane.07" ] } }, "rationale_derived": "Floored at 2 by humane.07 (M.2): Methods published.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by humane.07 (M.2): Methods published.. No admissible signal caps it." }, "X": { "evaluator": "humane", "dimension": "X", "value": 2, "anchor": 2, "signals": [ "humane.08", "humane.09" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "resolution": { "rule": "X.1", "value": 2, "note": "humane.08's raw floor of 4 is held to 3 by C15/C16 and still exceeds the X.1 cap; X.1 decides because the paid-services statement is on the organization's own page." }, "open_questions": [ "Request to Humane Intelligence: a list of paid red-teaming and contextual-evaluation clients since 2024, or a statement of whether any client is a frontier developer." ], "graph_id": "assessment#698", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "humane.09" ], "h": true, "c": true, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "humane.09" ], "h": true, "c": true, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "humane.09" ], "h": false, "c": false, "u": false, "x": [ "humane.08" ] }, "spans": { "v": 2, "b": [ "humane.09" ], "h": false, "c": false, "u": false, "x": [ "humane.08" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "humane.08", "humane.09" ] } }, "rationale_derived": "Conflict: floor 3 from humane.08 (X.7) against cap 2 from humane.09 (X.1); resolved at 2 by X.1: humane.08's raw floor of 4 is held to 3 by C15/C16 and still exceeds the X.1 cap; X.1 decides because the paid-services statement is on the organization's own page.", "rationale_leads": "Conflict: floor 3 from humane.08 (X.7) against cap 2 from humane.09 (X.1); resolved at 2 by X.1: humane.08's raw floor of 4 is held to 3 by C15/C16 and still exceeds the X.1 cap; X.1 decides because the paid-services statement is on the organization's own page." } }, "signals": [ { "id": "humane.01", "evaluator": "humane", "dimension": "R", "direction": "for", "claim": "Publishes openly; public exercises.", "sources": [ "humane-int" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "R.5", "graph_id": "signal#399", "source_graph_ids": [ "source#97" ] }, { "id": "humane.02", "evaluator": "humane", "dimension": "F", "direction": "for", "claim": "Government partners rather than lab clients.", "sources": [ "humane-int" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "F.9", "quote": "Humane Intelligence would not be able to do its important work without the support of our funders", "quote_source": "humane-int", "graph_id": "signal#400", "source_graph_ids": [ "source#97" ] }, { "id": "humane.03", "evaluator": "humane", "dimension": "A", "direction": "against", "claim": "Public-model access only.", "sources": [ "humane-int" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 0 }, "rule": "A.6", "graph_id": "signal#401", "source_graph_ids": [ "source#97" ] }, { "id": "humane.04", "evaluator": "humane", "dimension": "G", "direction": "for", "claim": "Nonprofit.", "sources": [ "humane-int" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "G.1", "quote": "Humane Intelligence is a 501(c)(3) nonprofit dedicated to breaking down barriers to AI deployment for social good", "quote_source": "humane-int", "graph_id": "signal#402", "source_graph_ids": [ "source#97" ] }, { "id": "humane.05", "evaluator": "humane", "dimension": "P", "direction": "for", "claim": "No lab roles found.", "sources": [ "humane-int" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "P.6", "graph_id": "signal#403", "source_graph_ids": [ "source#97" ] }, { "id": "humane.06", "evaluator": "humane", "dimension": "S", "direction": "for", "claim": "Designs its own exercises.", "sources": [ "humane-int" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "S.5", "quote": "We collaboratively design and run rigorous evaluations", "quote_source": "humane-int", "graph_id": "signal#404", "source_graph_ids": [ "source#97" ] }, { "id": "humane.07", "evaluator": "humane", "dimension": "M", "direction": "for", "claim": "Methods published.", "sources": [ "humane-int" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "M.2", "quote": "using our own software, which we are releasing under an open source software license in 2026", "quote_source": "humane-int", "graph_id": "signal#405", "source_graph_ids": [ "source#97" ] }, { "id": "humane.08", "evaluator": "humane", "dimension": "X", "direction": "for", "claim": "No commercial products.", "sources": [ "humane-int" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 4 }, "rule": "X.7", "graph_id": "signal#406", "source_graph_ids": [ "source#97" ] }, { "id": "humane.09", "evaluator": "humane", "dimension": "X", "direction": "against", "claim": "Humane Intelligence sells red-teaming events and contextual evaluations as paid services using its own software, and invites clients to 'hire us'; the client list is not published.", "sources": [ "humane-int" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 2 }, "rule": "X.1", "quote": "Humane Intelligence offers red teaming events as a paid service using our own software", "quote_source": "humane-int", "note": "Products or services sold to unspecified parties in the evaluated ecosystem cap at 2 until a lab customer is documented (X.1); the card carries a document request for the client list. Also on the page: 'Humane Intelligence designs and runs contextual evals as a paid service'.", "graph_id": "signal#407", "source_graph_ids": [ "source#97" ] }, { "id": "humane.10", "evaluator": "humane", "dimension": "P", "direction": "against", "claim": "Humane Intelligence's advisory group includes a Senior Research Scientist at Meta, and its board president previously worked on OpenAI's Human Data team; no recusal or cooling-off rule is published.", "sources": [ "humane-board" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 2 }, "rule": "P.4", "quote": "Dr. Diego Garcia-Olano Senior Research Scientist, Meta", "quote_source": "humane-board", "note": "An advisor employed by a lab caps at 2 regardless of recusal (P.4); a director hired from a lab without a cooling-off rule also caps at 2 (P.3; 'He was previously a member of OpenAI’s Human Data team'). Public roles only; the open question is a document request.", "graph_id": "signal#408", "source_graph_ids": [ "source#96" ] } ], "scores": { "lab": 57, "regulator": 57, "public": 63, "equal": 56 }, "scores_by_policy": { "leads": { "lab": { "score": 57, "band": "conditional", "coverage": 8, "range": [ 57, 57 ] }, "regulator": { "score": 57, "band": "conditional", "coverage": 8, "range": [ 57, 57 ] }, "public": { "score": 63, "band": "conditional", "coverage": 8, "range": [ 63, 63 ] }, "equal": { "score": 56, "band": "conditional", "coverage": 8, "range": [ 56, 56 ] } }, "standard": { "lab": { "score": 57, "band": "conditional", "coverage": 8, "range": [ 57, 57 ] }, "regulator": { "score": 57, "band": "conditional", "coverage": 8, "range": [ 57, 57 ] }, "public": { "score": 63, "band": "conditional", "coverage": 8, "range": [ 63, 63 ] }, "equal": { "score": 56, "band": "conditional", "coverage": 8, "range": [ 56, 56 ] } }, "against_interest": { "lab": { "score": 36, "band": "conditional", "coverage": 3, "range": [ 13, 78 ] }, "regulator": { "score": 33, "band": "conditional", "coverage": 3, "range": [ 10, 80 ] }, "public": { "score": 45, "band": "conditional", "coverage": 3, "range": [ 11, 86 ] }, "equal": { "score": 42, "band": "conditional", "coverage": 3, "range": [ 16, 78 ] } }, "spans": { "lab": { "score": 63, "band": "clear", "coverage": 6, "range": [ 42, 76 ] }, "regulator": { "score": 63, "band": "clear", "coverage": 6, "range": [ 41, 76 ] }, "public": { "score": 62, "band": "clear", "coverage": 6, "range": [ 46, 71 ] }, "equal": { "score": 58, "band": "clear", "coverage": 6, "range": [ 44, 69 ] } }, "primary": { "lab": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "regulator": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "public": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "equal": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] } } }, "band": "conditional", "coverage": 8, "floor": "A", "what_moves": { "lab": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 62, "band": "conditional", "base": 57 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 57 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 60, "band": "conditional", "base": 57 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 62, "band": "clear", "base": 57 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 61, "band": "conditional", "base": 57 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 61, "band": "conditional", "base": 57 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 57 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 58, "band": "conditional", "base": 57 } ], "regulator": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 61, "band": "conditional", "base": 57 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 60, "band": "conditional", "base": 57 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 60, "band": "conditional", "base": 57 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 63, "band": "clear", "base": 57 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 63, "band": "conditional", "base": 57 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 61, "band": "conditional", "base": 57 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 60, "band": "conditional", "base": 57 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 57, "band": "conditional", "base": 57 } ], "public": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 69, "band": "conditional", "base": 63 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 66, "band": "conditional", "base": 63 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 66, "band": "conditional", "base": 63 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 64, "band": "clear", "base": 63 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 65, "band": "conditional", "base": 63 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 68, "band": "conditional", "base": 63 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 64, "band": "conditional", "base": 63 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 64, "band": "conditional", "base": 63 } ], "equal": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 59, "band": "clear", "base": 56 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 56 } ] } }, { "id": "irregular", "name": "Irregular (formerly Pattern Labs)", "type": "vc", "hq": "Tel Aviv, IL / San Francisco, US", "domains": [ "cyber", "misuse" ], "confidence": "high", "summary": "Offensive-cyber evaluator cited in OpenAI and Anthropic system cards; SOLVE framework used by UK AISI. Pattern Labs and Irregular are the same company.", "what_would_move_the_score": "A published COI policy, disclosure of lab revenue share, and a firewall between evaluations and the security-products business.", "role": "vendor", "dissent": { "lower": "Role incompatibility (1 to 0). Irregular says it works side by side with OpenAI and Anthropic 'to develop the defenses needed before deployment' and describes a roadmap of runtime security controls for the same customers it evaluates. If any evaluated lab pays for those defenses, X.1 places the company at 0: it would be selling the fix to the party it grades.", "higher": "Funding (1 to 2). The revenue disclosed is 'millions' from lab partnerships, but the UK government uses the SOLVE framework and the company markets to enterprises beyond the labs. If a non-lab revenue base is documented, per-engagement lab fees beside that base cap at 2 under F.3 rather than 1 under F.5." }, "list_group": "commercial", "graph_id": "evaluator#217", "values": { "F": 1, "G": 1, "P": 2, "A": 2, "S": 2, "R": 2, "M": 2, "X": 1 }, "values_by_policy": { "leads": { "F": 1, "G": 1, "P": 2, "A": 2, "S": 2, "R": 2, "M": 2, "X": 1 }, "standard": { "F": 1, "G": 1, "P": 2, "A": 2, "S": 2, "R": 2, "M": 2, "X": 1 }, "against_interest": { "F": 1, "G": 1, "P": 2, "A": 2, "S": 2, "R": 2, "M": 2, "X": 1 }, "spans": { "F": 1, "G": 1, "P": 2, "A": 2, "S": 2, "R": 2, "M": 2, "X": 1 }, "primary": { "F": null, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "irregular", "dimension": "F", "value": 1, "anchor": 1, "signals": [ "irregular.01", "irregular.02", "irregular.06", "irregular.11", "irregular.12" ], "rationale": "Anchor 1: a $6.8M Coefficient Giving grant appears in the index under the former name Pattern Labs (irregular.11; ledger T09 marked differs, unreconciled). A name-search under ‘Irregular’ found nothing (irregular.12), which does not rebut the old-name row — the project treats the two names as the same company. Material lab-adjacent funding with disputed attribution.", "assessed": "2026-09-15", "assessor": "yohei/claude v1.0", "open_questions": [ "Re-run the Coefficient index search under both Irregular and Pattern Labs and record the snapshot." ], "graph_id": "assessment#699", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 1, "b": [ "irregular.01", "irregular.02" ], "h": false, "c": false, "u": false, "x": [ "irregular.11" ] }, "standard": { "v": 1, "b": [ "irregular.01", "irregular.02" ], "h": false, "c": false, "u": false, "x": [ "irregular.11" ] }, "against_interest": { "v": 1, "b": [ "irregular.01", "irregular.02" ], "h": false, "c": false, "u": false, "x": [ "irregular.11" ] }, "spans": { "v": 1, "b": [ "irregular.01", "irregular.02" ], "h": false, "c": false, "u": false, "x": [ "irregular.11", "irregular.12" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "irregular.01", "irregular.02", "irregular.06", "irregular.11", "irregular.12" ] } }, "rationale_derived": "Capped at 1 by irregular.01 (F.4): $80M raised in 2025 at a $450M valuation, led by Sequoia and Redpoint; Sequoia is also a major OpenAI investor.; capped at 1 by irregular.02 (F.5): Millions in annual revenue from the labs it evaluates.. Floors up to 1 from irregular.06 do not exceed the cap. Not counted under this policy: irregular.11 (superseded).", "rationale_leads": "Capped at 1 by irregular.01 (F.4): $80M raised in 2025 at a $450M valuation, led by Sequoia and Redpoint; Sequoia is also a major OpenAI investor.; capped at 1 by irregular.02 (F.5): Millions in annual revenue from the labs it evaluates.. Floors up to 1 from irregular.06 do not exceed the cap." }, "G": { "evaluator": "irregular", "dimension": "G", "value": 1, "anchor": 1, "signals": [ "irregular.03" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#700", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 1, "b": [ "irregular.03" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "irregular.03" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "irregular.03" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "irregular.03" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "irregular.03" ] } }, "rationale_derived": "Capped at 1 by irregular.03 (G.2): Venture-backed for-profit; no published COI or recusal policy found..", "rationale_leads": "Capped at 1 by irregular.03 (G.2): Venture-backed for-profit; no published COI or recusal policy found.." }, "P": { "evaluator": "irregular", "dimension": "P", "value": 2, "anchor": 2, "signals": [ "irregular.10" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#701", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "irregular.10" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "irregular.10" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "irregular.10" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "irregular.10" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "irregular.10" ] } }, "rationale_derived": "Capped at 2 by irregular.10 (P.5): Markets itself as embedded with the labs; no recusal or cooling-off policy found..", "rationale_leads": "Capped at 2 by irregular.10 (P.5): Markets itself as embedded with the labs; no recusal or cooling-off policy found.." }, "A": { "evaluator": "irregular", "dimension": "A", "value": 2, "anchor": 2, "signals": [ "irregular.05" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "lab-controlled", "graph_id": "assessment#702", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "irregular.05" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "irregular.05" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "irregular.05" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "irregular.05" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "irregular.05" ] } }, "rationale_derived": "Floored at 2 by irregular.05 (A.1): Deep, repeated pre-deployment access at OpenAI, Anthropic and Google DeepMind; embedded with labs.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by irregular.05 (A.1): Deep, repeated pre-deployment access at OpenAI, Anthropic and Google DeepMind; embedded with labs.. No admissible signal caps it." }, "S": { "evaluator": "irregular", "dimension": "S", "value": 2, "anchor": 2, "signals": [ "irregular.09" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#703", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "irregular.09" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "irregular.09" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "irregular.09" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "irregular.09" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "irregular.09" ] } }, "rationale_derived": "Capped at 2 by irregular.09 (S.2; S.6): Scope set by lab engagements; no public account of evaluator-set scope..", "rationale_leads": "Capped at 2 by irregular.09 (S.2; S.6): Scope set by lab engagements; no public account of evaluator-set scope.." }, "R": { "evaluator": "irregular", "dimension": "R", "value": 2, "anchor": 2, "signals": [ "irregular.08", "irregular.14" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "mechanism": "lab-controlled", "graph_id": "assessment#704", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "irregular.14" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "irregular.14" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "irregular.14" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "irregular.14" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "irregular.08", "irregular.14" ] } }, "rationale_derived": "Capped at 2 by irregular.14 (R.3): Findings on lab models reach the public only as lab-authored system-card citations; the company publishes its own research on other work (a RAND co-authored paper and an Anthropic co-authored whitepaper).. Floors up to 1 from irregular.08 do not exceed the cap.", "rationale_leads": "Capped at 2 by irregular.14 (R.3): Findings on lab models reach the public only as lab-authored system-card citations; the company publishes its own research on other work (a RAND co-authored paper and an Anthropic co-authored whitepaper).. Floors up to 1 from irregular.08 do not exceed the cap." }, "M": { "evaluator": "irregular", "dimension": "M", "value": 2, "anchor": 2, "signals": [ "irregular.07" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#705", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "irregular.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "irregular.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "irregular.07" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "irregular.07" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "irregular.07" ] } }, "rationale_derived": "Floored at 2 by irregular.07 (M.2): Publishes some method papers; SOLVE framework adopted by UK AISI and Anthropic.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by irregular.07 (M.2): Publishes some method papers; SOLVE framework adopted by UK AISI and Anthropic.. No admissible signal caps it." }, "X": { "evaluator": "irregular", "dimension": "X", "value": 1, "anchor": 1, "signals": [ "irregular.04", "irregular.13" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#706", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 1, "b": [ "irregular.13" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "irregular.13" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "irregular.13" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "irregular.13" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "irregular.04", "irregular.13" ] } }, "rationale_derived": "Capped at 1 by irregular.13 (X.2): Delivers paid evaluation services to OpenAI and Anthropic; lab partnerships are the only revenue the company discloses..", "rationale_leads": "Capped at 1 by irregular.13 (X.2): Delivers paid evaluation services to OpenAI and Anthropic; lab partnerships are the only revenue the company discloses.." } }, "signals": [ { "id": "irregular.01", "evaluator": "irregular", "dimension": "F", "direction": "against", "claim": "$80M raised in 2025 at a $450M valuation, led by Sequoia and Redpoint; Sequoia is also a major OpenAI investor.", "sources": [ "techcrunch-irregular", "sequoia-irregular", "irregular-xai-series-b" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "quote": "a round led by Sequoia Capital and Redpoint Ventures", "bound": { "cap": 1 }, "rule": "F.4", "quote_source": "techcrunch-irregular", "graph_id": "signal#409", "source_graph_ids": [ "source#178", "source#175", "source#100" ] }, { "id": "irregular.02", "evaluator": "irregular", "dimension": "F", "direction": "against", "claim": "Millions in annual revenue from the labs it evaluates.", "sources": [ "newswire-irregular" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 1 }, "rule": "F.5", "quote": "Already generating millions in revenue, Irregular partners with leading labs like OpenAI and Anthropic", "quote_source": "newswire-irregular", "graph_id": "signal#410", "source_graph_ids": [ "source#134" ] }, { "id": "irregular.03", "evaluator": "irregular", "dimension": "G", "direction": "against", "claim": "Venture-backed for-profit; no published COI or recusal policy found.", "sources": [ "techcrunch-irregular" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 1 }, "rule": "G.2", "quote": "a round led by Sequoia Capital and Redpoint Ventures", "quote_source": "techcrunch-irregular", "graph_id": "signal#411", "source_graph_ids": [ "source#178" ] }, { "id": "irregular.04", "evaluator": "irregular", "dimension": "X", "direction": "against", "claim": "Growth plan is to sell runtime security controls into the same customers it evaluates.", "sources": [ "newswire-irregular" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "X.1 (second sentence)", "quote": "to develop the defenses needed before deployment", "quote_source": "newswire-irregular", "graph_id": "signal#412", "source_graph_ids": [ "source#134" ] }, { "id": "irregular.05", "evaluator": "irregular", "dimension": "A", "direction": "for", "claim": "Deep, repeated pre-deployment access at OpenAI, Anthropic and Google DeepMind; embedded with labs.", "sources": [ "sequoia-irregular" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "A.1", "quote": "By embedding directly with these cutting-edge research labs", "quote_source": "sequoia-irregular", "graph_id": "signal#413", "source_graph_ids": [ "source#175" ] }, { "id": "irregular.06", "evaluator": "irregular", "dimension": "F", "direction": "for", "claim": "Government clients (UK) alongside labs.", "sources": [ "newswire-irregular" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 1 }, "rule": "F.5 (anchor text)", "quote": "the UK government and Anthropic use Irregular's SOLVE framework", "quote_source": "newswire-irregular", "graph_id": "signal#414", "source_graph_ids": [ "source#134" ] }, { "id": "irregular.07", "evaluator": "irregular", "dimension": "M", "direction": "for", "claim": "Publishes some method papers; SOLVE framework adopted by UK AISI and Anthropic.", "sources": [ "sequoia-irregular" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "M.2", "quote": "the UK government and Anthropic both use the company’s SOLVE framework", "quote_source": "sequoia-irregular", "graph_id": "signal#415", "source_graph_ids": [ "source#175" ] }, { "id": "irregular.08", "evaluator": "irregular", "dimension": "R", "direction": "for", "claim": "Findings appear in system cards for o3, o4-mini, GPT-5 and Claude models.", "sources": [ "techcrunch-irregular", "sequoia-irregular" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 1 }, "rule": "R.3 (anchor 1)", "quote": "Irregular’s evaluations are cited (under their former name, Pattern Labs) in system cards for GPT-4, o3, o4 mini and 5", "quote_source": "sequoia-irregular", "graph_id": "signal#416", "source_graph_ids": [ "source#178", "source#175" ] }, { "id": "irregular.09", "evaluator": "irregular", "dimension": "S", "direction": "against", "claim": "Scope set by lab engagements; no public account of evaluator-set scope.", "sources": [ "sequoia-irregular", "newswire-irregular" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "S.2; S.6", "quote": "The company also co-authored a whitepaper with Anthropic", "quote_source": "newswire-irregular", "graph_id": "signal#417", "source_graph_ids": [ "source#175", "source#134" ] }, { "id": "irregular.10", "evaluator": "irregular", "dimension": "P", "direction": "against", "claim": "Markets itself as embedded with the labs; no recusal or cooling-off policy found.", "sources": [ "sequoia-irregular" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "P.5", "quote": "By embedding directly with these cutting-edge research labs", "quote_source": "sequoia-irregular", "graph_id": "signal#418", "source_graph_ids": [ "source#175" ] }, { "id": "irregular.11", "evaluator": "irregular", "dimension": "F", "direction": "against", "claim": "A Coefficient Giving grant of about $6.8M appears in the index for a venture-backed company; purpose to be verified.", "sources": [ "coefficient-index" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-11", "superseded_by": "irregular.12", "bound": null, "rule": "0 (superseded)", "bound_note": "Superseded signals support nothing (RULES 0).", "graph_id": "signal#419", "source_graph_ids": [ "source#52" ] }, { "id": "irregular.12", "evaluator": "irregular", "dimension": "F", "direction": "for", "claim": "A search of Coefficient Giving's grants pages under the name Irregular found no grant (snapshot 2026-09-15); the imported row citing $6.8M under Pattern Labs is unreconciled and marked differs.", "sources": [ "coefficient-home", "evaluators-ledger" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2026-09-15", "bound": null, "rule": "F.6 (informational)", "bound_note": "Informational: neither floor nor cap; recorded as a gap.", "graph_id": "signal#420", "source_graph_ids": [ "source#51", "source#81" ] }, { "id": "irregular.13", "evaluator": "irregular", "dimension": "X", "direction": "against", "claim": "Delivers paid evaluation services to OpenAI and Anthropic; lab partnerships are the only revenue the company discloses.", "sources": [ "newswire-irregular" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 1 }, "rule": "X.2", "quote": "Already generating millions in revenue, Irregular partners with leading labs like OpenAI and Anthropic", "quote_source": "newswire-irregular", "note": "Needed to apply X.2 (evaluation services delivered to an evaluated lab for money: 1); irregular.04 concerns the planned defense product and caps only at 2.", "graph_id": "signal#421", "source_graph_ids": [ "source#134" ] }, { "id": "irregular.14", "evaluator": "irregular", "dimension": "R", "direction": "against", "claim": "Findings on lab models reach the public only as lab-authored system-card citations; the company publishes its own research on other work (a RAND co-authored paper and an Anthropic co-authored whitepaper).", "sources": [ "sequoia-irregular", "newswire-irregular" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 2 }, "rule": "R.3", "quote": "Irregular’s evaluations are cited (under their former name, Pattern Labs) in system cards for GPT-4, o3, o4 mini and 5", "quote_source": "sequoia-irregular", "note": "Needed to apply R.3: system-card-only publication caps at 1, or at 2 where the organization also publishes its own reports on other work (Newswire: 'The company also co-authored a whitepaper with Anthropic'; RAND paper).", "graph_id": "signal#422", "source_graph_ids": [ "source#175", "source#134" ] } ], "scores": { "lab": 42, "regulator": 44, "public": 39, "equal": 41 }, "scores_by_policy": { "leads": { "lab": { "score": 42, "band": "conditional", "coverage": 8, "range": [ 42, 42 ] }, "regulator": { "score": 44, "band": "conditional", "coverage": 8, "range": [ 44, 44 ] }, "public": { "score": 39, "band": "conditional", "coverage": 8, "range": [ 39, 39 ] }, "equal": { "score": 41, "band": "conditional", "coverage": 8, "range": [ 41, 41 ] } }, "standard": { "lab": { "score": 42, "band": "conditional", "coverage": 8, "range": [ 42, 42 ] }, "regulator": { "score": 44, "band": "conditional", "coverage": 8, "range": [ 44, 44 ] }, "public": { "score": 39, "band": "conditional", "coverage": 8, "range": [ 39, 39 ] }, "equal": { "score": 41, "band": "conditional", "coverage": 8, "range": [ 41, 41 ] } }, "against_interest": { "lab": { "score": 42, "band": "conditional", "coverage": 8, "range": [ 42, 42 ] }, "regulator": { "score": 44, "band": "conditional", "coverage": 8, "range": [ 44, 44 ] }, "public": { "score": 39, "band": "conditional", "coverage": 8, "range": [ 39, 39 ] }, "equal": { "score": 41, "band": "conditional", "coverage": 8, "range": [ 41, 41 ] } }, "spans": { "lab": { "score": 42, "band": "conditional", "coverage": 8, "range": [ 42, 42 ] }, "regulator": { "score": 44, "band": "conditional", "coverage": 8, "range": [ 44, 44 ] }, "public": { "score": 39, "band": "conditional", "coverage": 8, "range": [ 39, 39 ] }, "equal": { "score": 41, "band": "conditional", "coverage": 8, "range": [ 41, 41 ] } }, "primary": { "lab": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "regulator": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "public": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "equal": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] } } }, "band": "conditional", "coverage": 8, "floor": "F", "what_moves": { "lab": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 47, "band": "conditional", "base": 42 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 44, "band": "conditional", "base": 42 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 45, "band": "conditional", "base": 42 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 47, "band": "conditional", "base": 42 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 46, "band": "conditional", "base": 42 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 46, "band": "conditional", "base": 42 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 45, "band": "conditional", "base": 42 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 44, "band": "conditional", "base": 42 } ], "regulator": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 48, "band": "conditional", "base": 44 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 46, "band": "conditional", "base": 44 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 46, "band": "conditional", "base": 44 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 49, "band": "conditional", "base": 44 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 49, "band": "conditional", "base": 44 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 48, "band": "conditional", "base": 44 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 46, "band": "conditional", "base": 44 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 44, "band": "conditional", "base": 44 } ], "public": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 45, "band": "conditional", "base": 39 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 43, "band": "conditional", "base": 39 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 43, "band": "conditional", "base": 39 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 40, "band": "conditional", "base": 39 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 41, "band": "conditional", "base": 39 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 44, "band": "conditional", "base": 39 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 40, "band": "conditional", "base": 39 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 40, "band": "conditional", "base": 39 } ], "equal": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 44, "band": "conditional", "base": 41 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 44, "band": "conditional", "base": 41 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 44, "band": "conditional", "base": 41 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 44, "band": "conditional", "base": 41 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 44, "band": "conditional", "base": 41 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 44, "band": "conditional", "base": 41 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 44, "band": "conditional", "base": 41 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 44, "band": "conditional", "base": 41 } ] } }, { "id": "metr", "name": "METR", "type": "nonprofit", "hq": "Berkeley, US", "domains": [ "autonomy", "scheming", "incident", "assurance" ], "confidence": "high", "summary": "Reference evaluator for autonomy and AI R&D capability. Named by Anthropic as the model for embedded evaluators in September 2026; led the on-site investigation of the OpenAI agent swarm incident.", "what_would_move_the_score": "Contract terms for the Anthropic embedded program that fix scope-setting and publication rights in writing, plus a public cooling-off policy.", "role": "referee", "dissent": { "lower": "Personnel could be 1. A METR director founded FAR.AI, which OpenAI pays for red-teaming; a METR advisor is an a16z partner while a16z backs xAI and Thinking Machines; another advisor advises Thinking Machines; former advisors moved to Anthropic and to the OpenAI Foundation board; a former Anthropic researcher joined in September 2026. If advisors count as leaders, P.2 caps advisory roles at labs at 1, and the August 2026 policy has no cooling-off period.", "higher": "Personnel could be 3. The August 2026 conflict-of-interest policy requires staff disclosure before any company-identifying risk assessment, applies provisional recusal, bans direct equity in frontier AI companies, and bars frontier-company employees from the board; advisor and director affiliations are listed on the team page. Anchor 3 asks for a recusal policy and disclosure of lab ties, and both are now published; the one-hop seats are advisory, not lab roles." }, "list_group": "referee", "graph_id": "evaluator#218", "values": { "F": 3, "G": 3, "P": 2, "A": 4, "S": 3, "R": 3, "M": 3, "X": 4 }, "values_by_policy": { "leads": { "F": 3, "G": 3, "P": 2, "A": 4, "S": 3, "R": 3, "M": 3, "X": 4 }, "standard": { "F": 3, "G": 3, "P": 2, "A": 4, "S": 3, "R": 3, "M": 3, "X": 4 }, "against_interest": { "F": 3, "G": null, "P": 2, "A": 4, "S": 3, "R": 3, "M": null, "X": 4 }, "spans": { "F": 3, "G": 3, "P": 2, "A": 4, "S": 3, "R": 3, "M": 3, "X": 4 }, "primary": { "F": 3, "G": null, "P": 2, "A": null, "S": null, "R": null, "M": null, "X": 4 } }, "assessments": { "F": { "evaluator": "metr", "dimension": "F", "value": 3, "anchor": 3, "signals": [ "metr.01", "metr.02", "metr.03", "metr.14", "metr.15", "metr.16", "metr.18", "metr.19", "metr.20", "metr.21" ], "rationale": "Anchor 3: mostly philanthropic, no direct lab cash, but pooled and donor-advised funds with non-public donors, recommendations funded by a lab investor, in-kind credits from a lab, and a donor rule that is under two years old and still moving. Anchor 4 requires a confirmed bounded negative in primary filings and a resolved funder-exposure computation; both are on file as imported rows only.", "assessed": "2026-09-15", "assessor": "yohei/claude v1.0", "open_questions": [ "Request from METR the date and vehicle of the gift acknowledged on metr.org/about under \"many others, such as David Farhi\", and the text of the donor rule in force on that date; request from the donor a copy of the acknowledgment letter. (Replaces the fourth F open question, which framed the timing of a named person's gift as unresolved.)", "Request the Schedule I pages of the Vanguard Charitable and Audacious Project filings naming METR, and the Coefficient index rows checked, so that N01 to N05 can move from imported to confirmed." ], "graph_id": "assessment#707", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 3, "b": [ "metr.14", "metr.15", "metr.20", "metr.21" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "metr.15", "metr.20" ], "h": false, "c": false, "u": false, "x": [ "metr.14", "metr.21" ] }, "against_interest": { "v": 3, "b": [ "metr.15", "metr.20" ], "h": false, "c": false, "u": false, "x": [ "metr.14", "metr.19", "metr.21" ] }, "spans": { "v": 3, "b": [ "metr.15", "metr.20" ], "h": false, "c": false, "u": false, "x": [ "metr.14", "metr.21" ] }, "primary": { "v": 3, "b": [ "metr.15" ], "h": false, "c": false, "u": false, "x": [ "metr.01", "metr.02", "metr.03", "metr.14", "metr.19", "metr.20", "metr.21" ] } }, "rationale_derived": "Capped at 3 by metr.15 (F.6): Traced inflows in the ledger: no direct grant from Coefficient Giving in its index, but the 2024 spin-out transfer from ARC (seeded by Coefficient and SFF), SFF recommendations funded by an Anthropic Series A investor, a Longview pooled-fund grant, and about $21M through donor-advised funds and the Audacious Project whose underlying donors are not public.; capped at 3 by metr.20 (F.6): The Audacious Project committed approximately $17M to METR for Canary work. The $38M combined figure and the funder-collective membership are per the cited sources; spans not yet captured.. Floors up to 3 from metr.01, metr.02, metr.16, metr.18, metr.19 do not exceed the cap. Not counted under this policy: metr.14 (no confirmed source (unverifiable)); metr.21 (no confirmed source (unaudited)).", "rationale_leads": "Capped at 3 by metr.14 (F.8): The no-lab-money rule is recent and its wording has moved: absent from the April 2024 page, a compensation footnote in April 2025, 'has not accepted funding from AI companies' with free-credit language from August 2025, and a September 2025 footnote that donations from individual lab employees are accepted. An OpenAI technical lead is named among individual donors.; capped at 3 by metr.15 (F.6): Traced inflows in the ledger: no direct grant from Coefficient Giving in its index, but the 2024 spin-out transfer from ARC (seeded by Coefficient and SFF), SFF recommendations funded by an Anthropic Series A investor, a Longview pooled-fund grant, and about $21M through donor-advised funds and the Audacious Project whose underlying donors are not public.; capped at 3 by metr.20 (F.6): The Audacious Project committed approximately $17M to METR for Canary work. The $38M combined figure and the funder-collective membership are per the cited sources; spans not yet captured.; capped at 3 by metr.21 (F.6): Second-hop routes for METR's philanthropic money: ARC (a $4,553,935 grant to METR in its 2024 filing, per a 990 extraction) was itself funded by Coefficient; Longview (a pooled fund named by METR) received a $15,961,273 operational grant from Coefficient; Constellation, which shares people with METR, received $16,750,000.. Floors up to 3 from metr.01, metr.02, metr.16, metr.18, metr.19 do not exceed the cap." }, "G": { "evaluator": "metr", "dimension": "G", "value": 3, "anchor": 3, "signals": [ "metr.10" ], "rationale": "Anchor 3: nonprofit with a written independence policy and a no-lab-money rule (metr.10). Independent board and external review are not evidenced at tier 1, so 4 is unearned under the tier rule (C10).", "assessed": "2026-09-15", "assessor": "yohei/claude v0", "graph_id": "assessment#708", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "metr.10" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "metr.10" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "metr.10" ] }, "spans": { "v": 3, "b": [ "metr.10" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "metr.10" ] } }, "rationale_derived": "Floored at 3 by metr.10 (G.2): Nonprofit with a written independence policy and a no-lab-money rule.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by metr.10 (G.2): Nonprofit with a written independence policy and a no-lab-money rule.. No admissible signal caps it." }, "P": { "evaluator": "metr", "dimension": "P", "value": 2, "anchor": 2, "signals": [ "metr.09", "metr.17", "metr.23" ], "rationale": "Anchor 2: two-way movement between METR and labs, and board or advisor seats one to two steps from labs; no published recusal or cooling-off policy found. Sources are imported and need re-derivation from metr.org/team captures and the FY2024 990.", "assessed": "2026-09-14", "assessor": "yohei/claude v0.3", "open_questions": [ "Request from METR the August 2026 policy's treatment of post-employment cooling-off for hires from frontier AI companies (none appears in version 1.0) and dated captures of the team page showing current versus former advisor roles." ], "resolution": { "rule": "P.4", "value": 2, "note": "P.4 caps at 2 regardless of recusal; P.3 concurs because no cooling-off rule exists. The P.5 floor from the policy is real but the caps decide." }, "graph_id": "assessment#709", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 2, "b": [ "metr.09", "metr.17" ], "h": false, "c": true, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "metr.09", "metr.17" ], "h": false, "c": true, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "metr.09", "metr.17" ], "h": false, "c": false, "u": false, "x": [ "metr.23" ] }, "spans": { "v": 2, "b": [ "metr.09", "metr.17" ], "h": false, "c": true, "u": false, "x": [] }, "primary": { "v": 2, "b": [ "metr.17" ], "h": false, "c": false, "u": false, "x": [ "metr.09", "metr.23" ] } }, "rationale_derived": "Conflict: floor 3 from metr.23 (P.5) against cap 2 from metr.09, metr.17 (P.3, P.4); resolved at 2 by P.4: P.4 caps at 2 regardless of recusal; P.3 concurs because no cooling-off rule exists. The P.5 floor from the policy is real but the caps decide.", "rationale_leads": "Conflict: floor 3 from metr.23 (P.5) against cap 2 from metr.09, metr.17 (P.3, P.4); resolved at 2 by P.4: P.4 caps at 2 regardless of recusal; P.3 concurs because no cooling-off rule exists. The P.5 floor from the policy is real but the caps decide." }, "A": { "evaluator": "metr", "dimension": "A", "value": 4, "anchor": 4, "signals": [ "metr.04", "metr.05", "metr.13" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "mechanism": "lab-controlled", "graph_id": "assessment#710", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 4, "b": [ "metr.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 4, "b": [ "metr.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 4, "b": [ "metr.04" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 4, "b": [ "metr.04" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "metr.04", "metr.05", "metr.13" ] } }, "rationale_derived": "Floored at 4 by metr.04 (A.5): Investigated the OpenAI incident on site over six days with a Redwood contractor.. No admissible signal caps it.", "rationale_leads": "Floored at 4 by metr.04 (A.5): Investigated the OpenAI incident on site over six days with a Redwood contractor.. No admissible signal caps it." }, "S": { "evaluator": "metr", "dimension": "S", "value": 3, "anchor": 3, "signals": [ "metr.06" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#711", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "metr.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "metr.06" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "metr.06" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "metr.06" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "metr.06" ] } }, "rationale_derived": "Capped at 3 by metr.06 (S.1): OpenAI set the investigation window (June 26 to July 13); the lab's own cluster breach and three of METR's standard incident questions were out of scope..", "rationale_leads": "Capped at 3 by metr.06 (S.1): OpenAI set the investigation window (June 26 to July 13); the lab's own cluster breach and three of METR's standard incident questions were out of scope.." }, "R": { "evaluator": "metr", "dimension": "R", "value": 3, "anchor": 3, "signals": [ "metr.07", "metr.08" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "mechanism": "lab-controlled", "graph_id": "assessment#712", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "metr.08" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "metr.08" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "metr.08" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "metr.08" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "metr.07", "metr.08" ] } }, "rationale_derived": "Capped at 3 by metr.08 (R.2): OpenAI could redact any non-public information and gave feedback on structure, emphasis, clarity and tone, which METR incorporated.. Floors up to 2 from metr.07 do not exceed the cap.", "rationale_leads": "Capped at 3 by metr.08 (R.2): OpenAI could redact any non-public information and gave feedback on structure, emphasis, clarity and tone, which METR incorporated.. Floors up to 2 from metr.07 do not exceed the cap." }, "M": { "evaluator": "metr", "dimension": "M", "value": 3, "anchor": 3, "signals": [ "metr.11" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#713", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "metr.11" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "metr.11" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "metr.11" ] }, "spans": { "v": 3, "b": [ "metr.11" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "metr.11" ] } }, "rationale_derived": "Floored at 3 by metr.11 (M.1): Open task suites and published time-horizon methodology; results appear in system cards.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by metr.11 (M.1): Open task suites and published time-horizon methodology; results appear in system cards.. No admissible signal caps it." }, "X": { "evaluator": "metr", "dimension": "X", "value": 4, "anchor": 4, "signals": [ "metr.12", "metr.22" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#714", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 4, "b": [ "metr.22" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 4, "b": [ "metr.22" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 4, "b": [ "metr.22" ], "h": false, "c": false, "u": false, "x": [ "metr.12" ] }, "spans": { "v": 4, "b": [ "metr.22" ], "h": false, "c": false, "u": false, "x": [ "metr.12" ] }, "primary": { "v": 4, "b": [ "metr.22" ], "h": false, "c": false, "u": false, "x": [ "metr.12" ] } }, "rationale_derived": "Floored at 4 by metr.22 (X.7): FY2024 Form 990 (filed 16 November 2025) reports $0 program service revenue against $13.6M of contributions, and the August 2026 conflict-of-interest policy states METR has to date received no payment for work on company-identifying risk assessments; the only earned income named is a technical-assistance contract with the European AI Office.. No admissible signal caps it.", "rationale_leads": "Floored at 4 by metr.22 (X.7): FY2024 Form 990 (filed 16 November 2025) reports $0 program service revenue against $13.6M of contributions, and the August 2026 conflict-of-interest policy states METR has to date received no payment for work on company-identifying risk assessments; the only earned income named is a technical-assistance contract with the European AI Office.. No admissible signal caps it." } }, "signals": [ { "claim": "Hard rule against accepting money from AI companies, including donations directed by their staff; ~$71M raised in 2026 from foundations and individuals.", "curator": "yohei/claude v0", "dimension": "F", "direction": "for", "evaluator": "metr", "id": "metr.01", "recorded": "2026-09-14", "sources": [ "metr-about", "alphasignal-metr-71m", "metr-funding-2026" ], "bound": { "floor": 3 }, "rule": "F.8", "quote": "METR refuses funding from frontier AI companies and bans donations directed by their staff", "quote_source": "alphasignal-metr-71m", "graph_id": "signal#423", "source_graph_ids": [ "source#110", "source#16", "source#114" ] }, { "claim": "Funder base widened beyond the effective-altruism network, including Pew Charitable Trusts.", "curator": "yohei/claude v0", "dimension": "F", "direction": "for", "evaluator": "metr", "id": "metr.02", "recorded": "2026-09-14", "sources": [ "itbb-watchdogs" ], "bound": { "floor": 3 }, "rule": "F.12", "quote": "including the Pew Charitable Trusts and several foundations outside the usual network", "quote_source": "itbb-watchdogs", "graph_id": "signal#424", "source_graph_ids": [ "source#101" ] }, { "claim": "Runs on large volumes of free tokens and privileged access from the labs it evaluates.", "curator": "yohei/claude v0", "dimension": "F", "direction": "against", "evaluator": "metr", "id": "metr.03", "recorded": "2026-09-14", "sources": [ "metr-about" ], "bound": null, "rule": "F.11", "bound_note": "F.11: free tokens whose amount is undisclosed and which METR describes as significant but not as something it depends on carry no funding bound; the fact is recorded on the access mechanism (lab-controlled).", "quote": "though we make use of significant free tokens, which we use for evaluations, research, and engineering", "quote_source": "metr-about", "graph_id": "signal#425", "source_graph_ids": [ "source#110" ] }, { "claim": "Investigated the OpenAI incident on site over six days with a Redwood contractor.", "curator": "yohei/claude v0", "dimension": "A", "direction": "for", "evaluator": "metr", "id": "metr.04", "recorded": "2026-09-14", "sources": [ "metr-hf-investigation", "techtimes-metr-hf" ], "bound": { "floor": 4 }, "rule": "A.5", "quote": "worked on premises at OpenAI over a total of six days", "quote_source": "metr-hf-investigation", "graph_id": "signal#426", "source_graph_ids": [ "source#116", "source#182" ] }, { "claim": "Named by Anthropic as the example embedded evaluator: an ongoing, employee-like-access commitment with publication without editorial control; OpenAI's chief executive said OpenAI would do the same on access, without naming an evaluator or publication rights.", "curator": "yohei/claude v0", "dimension": "A", "direction": "for", "evaluator": "metr", "id": "metr.05", "recorded": "2026-09-14", "sources": [ "unite-altman-match", "marktechpost-pace", "anthropic-pace-frontier", "anthropic-amodei-x-pace-frontier", "openai-altman-embedded-pledge" ], "bound": null, "rule": "A.1", "bound_note": "A.1 and section 9: the Anthropic essay and the OpenAI match are pledges naming METR as the example embedded evaluator; no embedded engagement is documented as granted, so the signal is informational until access under the program is documented.", "quote": "a team of embedded third-party evaluators (such as METR )", "quote_source": "anthropic-pace-frontier", "as_of": "2026-09-12", "graph_id": "signal#427", "source_graph_ids": [ "source#198", "source#108", "source#27", "source#26", "source#142" ] }, { "claim": "OpenAI set the investigation window (June 26 to July 13); the lab's own cluster breach and three of METR's standard incident questions were out of scope.", "curator": "yohei/claude v0", "dimension": "S", "direction": "against", "evaluator": "metr", "id": "metr.06", "recorded": "2026-09-14", "sources": [ "pebblous-scope", "metr-hf-investigation" ], "bound": { "cap": 3 }, "rule": "S.1", "quote": "OpenAI defined the investigation period as June 26th through July 13th", "quote_source": "metr-hf-investigation", "graph_id": "signal#428", "source_graph_ids": [ "source#150", "source#116" ] }, { "claim": "Published its own 91-page report with a redaction summary statement; took no payment for the investigation.", "curator": "yohei/claude v0", "dimension": "R", "direction": "for", "evaluator": "metr", "id": "metr.07", "recorded": "2026-09-14", "sources": [ "metr-hf-investigation", "marktechpost-pace" ], "bound": { "floor": 2 }, "rule": "R.7", "quote": "Redaction summary statement: Except where explicitly noted in this post, OpenAI redacted no additional information", "quote_source": "metr-hf-investigation", "graph_id": "signal#429", "source_graph_ids": [ "source#116", "source#108" ] }, { "claim": "OpenAI could redact any non-public information and gave feedback on structure, emphasis, clarity and tone, which METR incorporated.", "curator": "yohei/claude v0", "dimension": "R", "direction": "against", "evaluator": "metr", "id": "metr.08", "recorded": "2026-09-14", "sources": [ "metr-hf-investigation" ], "bound": { "cap": 3 }, "rule": "R.2", "quote": "we made corrections and edits to structure, emphasis, clarity, and tone based on that feedback", "quote_source": "metr-hf-investigation", "graph_id": "signal#430", "source_graph_ids": [ "source#116" ] }, { "claim": "Hires from the labs it evaluates; a former Anthropic researcher joined in September 2026.", "curator": "yohei/claude v0", "dimension": "P", "direction": "against", "evaluator": "metr", "id": "metr.09", "recorded": "2026-09-14", "sources": [ "wapo-examiner-benton" ], "bound": { "cap": 2 }, "rule": "P.3", "quote": "Joe Benton, announced this week that he is joining the independent AI safety organization METR", "quote_source": "wapo-examiner-benton", "graph_id": "signal#431", "source_graph_ids": [ "source#199" ] }, { "claim": "Nonprofit with a written independence policy and a no-lab-money rule.", "curator": "yohei/claude v0", "dimension": "G", "direction": "for", "evaluator": "metr", "id": "metr.10", "recorded": "2026-09-14", "sources": [ "metr-about", "metr-coi-policy" ], "bound": { "floor": 3 }, "rule": "G.2", "quote": "Our current conflict of interest policy is here", "quote_source": "metr-about", "graph_id": "signal#432", "source_graph_ids": [ "source#110", "source#112" ] }, { "claim": "Open task suites and published time-horizon methodology; results appear in system cards.", "curator": "yohei/claude v0", "dimension": "M", "direction": "for", "evaluator": "metr", "id": "metr.11", "recorded": "2026-09-14", "sources": [ "metr-about", "metr-hcast-public" ], "bound": { "floor": 3 }, "rule": "M.1", "quote": "This repo contains source code for a subset of the tasks used in the HCAST", "quote_source": "metr-hcast-public", "graph_id": "signal#433", "source_graph_ids": [ "source#110", "source#115" ] }, { "claim": "No commercial products.", "curator": "yohei/claude v0", "dimension": "X", "direction": "for", "evaluator": "metr", "id": "metr.12", "recorded": "2026-09-14", "sources": [ "metr-about" ], "bound": { "floor": 3 }, "rule": "X.7", "graph_id": "signal#434", "source_graph_ids": [ "source#110" ] }, { "claim": "Access remains voluntary; no law requires any lab to grant it.", "curator": "yohei/claude v0", "dimension": "A", "direction": "against", "evaluator": "metr", "id": "metr.13", "recorded": "2026-09-14", "sources": [ "itbb-watchdogs" ], "bound": null, "rule": "A.3", "bound_note": "A.3: voluntary, revocable access is recorded as the mechanism (lab-controlled), not as a cap.", "quote": "that access is voluntary. Companies grant it because they choose to, not because any law or agreement requires it", "quote_source": "itbb-watchdogs", "graph_id": "signal#435", "source_graph_ids": [ "source#101" ] }, { "as_of": "2025-12", "claim": "The no-lab-money rule is recent and its wording has moved: absent from the April 2024 page, a compensation footnote in April 2025, 'has not accepted funding from AI companies' with free-credit language from August 2025, and a September 2025 footnote that donations from individual lab employees are accepted. An OpenAI technical lead is named among individual donors.", "curator": "yohei/claude v0.3", "dimension": "F", "direction": "against", "evaluator": "metr", "id": "metr.14", "recorded": "2026-09-14", "sources": [ "metr-donor-rule-history" ], "bound": { "cap": 3 }, "rule": "F.8", "graph_id": "signal#436", "source_graph_ids": [ "source#113" ] }, { "as_of": "2026-09-14", "claim": "Traced inflows in the ledger: no direct grant from Coefficient Giving in its index, but the 2024 spin-out transfer from ARC (seeded by Coefficient and SFF), SFF recommendations funded by an Anthropic Series A investor, a Longview pooled-fund grant, and about $21M through donor-advised funds and the Audacious Project whose underlying donors are not public.", "curator": "yohei/claude v0.3", "dimension": "F", "direction": "against", "evaluator": "metr", "id": "metr.15", "recorded": "2026-09-14", "sources": [ "coefficient-index", "metr-990-fy2024", "metr-money-figure" ], "bound": { "cap": 3 }, "rule": "F.6", "quote": "The accounts at SVCF and NPT that pay Coefficient-recommended grants are unattributed", "quote_source": "metr-money-figure", "graph_id": "signal#437", "source_graph_ids": [ "source#52", "source#109", "source#117" ] }, { "as_of": "2026-09-14", "claim": "Bounded negatives on file: no METR-named grant in Coefficient's 2,911-row index (snapshot 2026-09-11) or in the Good Ventures, Schmidt Sciences, Pew, and Packard filings checked; all imported and awaiting re-derivation.", "curator": "yohei/claude v0.3", "dimension": "F", "direction": "for", "evaluator": "metr", "id": "metr.16", "recorded": "2026-09-14", "sources": [ "coefficient-index", "metr-money-figure" ], "bound": { "floor": 3 }, "rule": "F.12", "quote": "No direct Coefficient grant to METR appears in the checked index and filings", "quote_source": "metr-money-figure", "graph_id": "signal#438", "source_graph_ids": [ "source#52", "source#117" ] }, { "as_of": "2026-09-14", "claim": "Board and advisors carry lab ties one or two steps out: a director who founded FAR.AI, which is paid by OpenAI for red-teaming; an advisor who is an a16z partner, the firm that led Thinking Machines' seed; an ex-OpenAI advisor who advises Thinking Machines; a director who leads an insurance-certification startup seeded by an Anthropic co-founder. Former advisors moved to Anthropic and to the OpenAI Foundation board.", "curator": "yohei/claude v0.3", "dimension": "P", "direction": "against", "evaluator": "metr", "id": "metr.17", "recorded": "2026-09-14", "sources": [ "metr-team", "metr-990-fy2024", "metr-money-figure" ], "bound": { "cap": 2 }, "rule": "P.4", "quote": "Adam Gleave (Director)", "quote_source": "metr-990-fy2024", "graph_id": "signal#439", "source_graph_ids": [ "source#119", "source#109", "source#117" ] }, { "as_of": "2026-09-14", "claim": "Current page (re-fetched 14 Sep 2026) states METR cannot accept donations made by or at the direction of frontier AI company employees, tighter than the September 2025 footnote; the SFF 2025 recommendation of $548,000 is confirmed on the fund's page and FY2024 revenue of $13.6M on ProPublica.", "curator": "yohei/claude v0.5", "dimension": "F", "direction": "for", "evaluator": "metr", "id": "metr.18", "quote": "METR has not accepted funding from AI companies", "recorded": "2026-09-14", "sources": [ "metr-about", "sff-2025", "propublica-metr-arc" ], "bound": { "floor": 3 }, "rule": "F.8", "quote_source": "metr-about", "graph_id": "signal#440", "source_graph_ids": [ "source#110", "source#176", "source#152" ] }, { "as_of": "2026-08-14", "claim": "METR raised commitments of about $71 million in the six months to August 2026, per its funding update.", "curator": "yohei/claude v1.0", "dimension": "F", "direction": "for", "evaluator": "metr", "id": "metr.19", "quote": "In the last 6 months, METR raised commitments of around $71 million", "recorded": "2026-09-15", "sources": [ "metr-funding-2026" ], "bound": { "floor": 3 }, "rule": "F.12", "quote_source": "metr-funding-2026", "graph_id": "signal#441", "source_graph_ids": [ "source#114" ] }, { "as_of": "2024-10-09", "claim": "The Audacious Project committed approximately $17M to METR for Canary work. The $38M combined figure and the funder-collective membership are per the cited sources; spans not yet captured.", "curator": "yohei/claude v1.0", "dimension": "F", "direction": "against", "evaluator": "metr", "id": "metr.20", "quote": "Approximately $17 million of this will support work at METR.", "recorded": "2026-09-15", "sources": [ "metr-audacious-2024" ], "bound": { "cap": 3 }, "rule": "F.6", "quote_source": "metr-audacious-2024", "graph_id": "signal#442", "source_graph_ids": [ "source#111" ] }, { "as_of": "2025", "claim": "Second-hop routes for METR's philanthropic money: ARC (a $4,553,935 grant to METR in its 2024 filing, per a 990 extraction) was itself funded by Coefficient; Longview (a pooled fund named by METR) received a $15,961,273 operational grant from Coefficient; Constellation, which shares people with METR, received $16,750,000.", "curator": "yohei/claude v1.0", "dimension": "F", "direction": "against", "evaluator": "metr", "id": "metr.21", "quote": "Total giving $4,553,935", "recorded": "2026-09-15", "sources": [ "instrumentl-arc-990", "op-longview", "op-constellation" ], "bound": { "cap": 3 }, "rule": "F.6", "quote_source": "instrumentl-arc-990", "graph_id": "signal#443", "source_graph_ids": [ "source#99", "source#138", "source#136" ] }, { "id": "metr.22", "evaluator": "metr", "dimension": "X", "direction": "for", "claim": "FY2024 Form 990 (filed 16 November 2025) reports $0 program service revenue against $13.6M of contributions, and the August 2026 conflict-of-interest policy states METR has to date received no payment for work on company-identifying risk assessments; the only earned income named is a technical-assistance contract with the European AI Office.", "sources": [ "metr-990-fy2024", "metr-coi-policy" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "floor": 4 }, "rule": "X.7", "quote": "Contributions $13,603,035 99.7% Program Services $0", "quote_source": "metr-990-fy2024", "note": "X.7 needs a bounded statement of no commercial products with a span and a second source: the filing (tier 1) supplies both the bounded negative and the provenance; the policy PDF supplies the organization's own statement (\"To date, we have not received payment for work on company-identifying risk assessments\"). A regulator technical-assistance contract is not a commercial product.", "graph_id": "signal#444", "source_graph_ids": [ "source#109", "source#112" ] }, { "id": "metr.23", "evaluator": "metr", "dimension": "P", "direction": "for", "claim": "Published conflict-of-interest policy (version 1.0, 28 August 2026): staff under consideration for a company-identifying risk assessment must disclose conflicts, tiered conflicts require external disclosure in the report, provisional recusal applies during review, staff may not hold direct equity or debt in frontier AI companies, and frontier-company employees are ineligible for the board. No post-employment cooling-off period.", "sources": [ "metr-coi-policy" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "floor": 3 }, "rule": "P.5", "quote": "Do not invest in direct equity or debt of frontier AI companies or reasonable economic proxies for specific companies", "quote_source": "metr-coi-policy", "note": "Mandatory recusal plus disclosure of individual ties floors at 3 (P.5). P.8 (4) needs a cooling-off rule, which the policy lacks, and tier-1 evidence. Span verified with pdftotext; the CI normalizer cannot read PDFs. Caps from metr.09 (P.3) and metr.17 (P.4) still decide the value; see conflicts.", "graph_id": "signal#445", "source_graph_ids": [ "source#112" ] } ], "scores": { "lab": 79, "regulator": 78, "public": 74, "equal": 78 }, "scores_by_policy": { "leads": { "lab": { "score": 79, "band": "clear", "coverage": 8, "range": [ 79, 79 ] }, "regulator": { "score": 78, "band": "clear", "coverage": 8, "range": [ 78, 78 ] }, "public": { "score": 74, "band": "clear", "coverage": 8, "range": [ 74, 74 ] }, "equal": { "score": 78, "band": "clear", "coverage": 8, "range": [ 78, 78 ] } }, "standard": { "lab": { "score": 79, "band": "clear", "coverage": 8, "range": [ 79, 79 ] }, "regulator": { "score": 78, "band": "clear", "coverage": 8, "range": [ 78, 78 ] }, "public": { "score": 74, "band": "clear", "coverage": 8, "range": [ 74, 74 ] }, "equal": { "score": 78, "band": "clear", "coverage": 8, "range": [ 78, 78 ] } }, "against_interest": { "lab": { "score": 80, "band": "clear", "coverage": 6, "range": [ 66, 83 ] }, "regulator": { "score": 78, "band": "clear", "coverage": 6, "range": [ 63, 83 ] }, "public": { "score": 73, "band": "clear", "coverage": 6, "range": [ 59, 79 ] }, "equal": { "score": 79, "band": "clear", "coverage": 6, "range": [ 59, 84 ] } }, "spans": { "lab": { "score": 79, "band": "clear", "coverage": 8, "range": [ 79, 79 ] }, "regulator": { "score": 78, "band": "clear", "coverage": 8, "range": [ 78, 78 ] }, "public": { "score": 74, "band": "clear", "coverage": 8, "range": [ 74, 74 ] }, "equal": { "score": 78, "band": "clear", "coverage": 8, "range": [ 78, 78 ] } }, "primary": { "lab": { "score": 71, "band": "clear", "coverage": 3, "range": [ 24, 91 ] }, "regulator": { "score": 65, "band": "clear", "coverage": 3, "range": [ 16, 91 ] }, "public": { "score": 69, "band": "clear", "coverage": 3, "range": [ 31, 86 ] }, "equal": { "score": 75, "band": "clear", "coverage": 3, "range": [ 28, 91 ] } } }, "band": "clear", "coverage": 8, "floor": "P", "what_moves": { "lab": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 83, "band": "clear", "base": 79 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 81, "band": "clear", "base": 79 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 81, "band": "clear", "base": 79 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 83, "band": "clear", "base": 79 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 82, "band": "clear", "base": 79 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 81, "band": "clear", "base": 79 } ], "regulator": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 81, "band": "clear", "base": 78 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 80, "band": "clear", "base": 78 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 80, "band": "clear", "base": 78 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 83, "band": "clear", "base": 78 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 81, "band": "clear", "base": 78 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 80, "band": "clear", "base": 78 } ], "public": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 80, "band": "clear", "base": 74 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 74 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 78, "band": "clear", "base": 74 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 76, "band": "clear", "base": 74 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 79, "band": "clear", "base": 74 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 74 } ], "equal": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 81, "band": "clear", "base": 78 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 81, "band": "clear", "base": 78 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 81, "band": "clear", "base": 78 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 81, "band": "clear", "base": 78 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 81, "band": "clear", "base": 78 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 81, "band": "clear", "base": 78 } ] } }, { "id": "mlcommons", "name": "MLCommons (AILuminate)", "type": "consortium", "hq": "San Francisco, US", "domains": [ "benchmarks", "assurance" ], "confidence": "med", "summary": "Industry consortium producing the AILuminate safety benchmark.", "what_would_move_the_score": "Published funding shares and a hazard suite the members do not see in advance.", "role": "benchmark", "dissent": { "lower": "Funding (F) at 1 could be 0. The consortium was convened by Google engineers, its president is a Google senior staff engineer, and Google is a founding participant; the benchmarked labs sit in the working group that writes the benchmark and pay the dues that fund it. Anchor 0 covers an organization controlled by a frontier developer; a member-governed body whose president draws a Google salary is arguably that.", "higher": "Funding (F) at 1 could be 2. More than 125 members and affiliates pay dues, most of them chipmakers, universities and non-profits rather than frontier labs, and no member is documented above a trivial share. The AI Verify Foundation co-funded AILuminate. Lab dues beside a broad disclosed base is the anchor-2 picture of diversified revenue rather than lab revenue as the principal source." }, "list_group": "commercial", "graph_id": "evaluator#219", "values": { "F": 1, "G": 2, "P": 2, "A": 0, "S": 3, "R": 2, "M": 3, "X": 3 }, "values_by_policy": { "leads": { "F": 1, "G": 2, "P": 2, "A": 0, "S": 3, "R": 2, "M": 3, "X": 3 }, "standard": { "F": 1, "G": 2, "P": 2, "A": 0, "S": 3, "R": 2, "M": 3, "X": 3 }, "against_interest": { "F": 1, "G": 2, "P": 2, "A": 0, "S": null, "R": null, "M": 3, "X": null }, "spans": { "F": 1, "G": 2, "P": 2, "A": 0, "S": 3, "R": 2, "M": 3, "X": 3 }, "primary": { "F": null, "G": 2, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "anchor": 1, "assessed": "2026-09-14", "assessor": "yohei/claude v0", "dimension": "F", "evaluator": "mlcommons", "rationale": "Anchor 1 on funding given the cited signals. Curator disclosure (2026-09-15): the curator holds small public-market shareholdings in Google and Meta. This evaluator has a confirmed ledger tie to Google (founding participation row T63; not a grant). Per the disclosure Rule, this assessment is flagged for public review: the rationale names the holding so readers can weigh it, and any reader can file a correction through the contribution path.", "signals": [ "mlcommons.01" ], "value": 1, "graph_id": "assessment#715", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 1, "b": [ "mlcommons.01" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "mlcommons.01" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "mlcommons.01" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "mlcommons.01" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "mlcommons.01" ] } }, "rationale_derived": "Capped at 1 by mlcommons.01 (F.5): Funded by member companies, including the labs whose models are benchmarked..", "rationale_leads": "Capped at 1 by mlcommons.01 (F.5): Funded by member companies, including the labs whose models are benchmarked.." }, "G": { "anchor": 2, "assessed": "2026-09-14", "assessor": "yohei/claude v0", "dimension": "G", "evaluator": "mlcommons", "signals": [ "mlcommons.04" ], "value": 2, "graph_id": "assessment#716", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 2, "b": [ "mlcommons.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "mlcommons.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "mlcommons.04" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "mlcommons.04" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": 2, "b": [ "mlcommons.04" ], "h": false, "c": false, "u": false, "x": [] } }, "rationale_derived": "Capped at 2 by mlcommons.04 (G.4): Consortium governance by members..", "rationale_leads": "Capped at 2 by mlcommons.04 (G.4): Consortium governance by members.." }, "P": { "anchor": 2, "assessed": "2026-09-14", "assessor": "yohei/claude v0", "dimension": "P", "evaluator": "mlcommons", "signals": [ "mlcommons.07" ], "value": 2, "open_questions": [ "Request from MLCommons the board conflict-of-interest and recusal policy that applies to directors employed by member companies whose systems are benchmarked, including any recusal recorded for the President on AILuminate grading decisions." ], "graph_id": "assessment#717", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "mlcommons.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "mlcommons.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "mlcommons.07" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "mlcommons.07" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "mlcommons.07" ] } }, "rationale_derived": "Capped at 2 by mlcommons.07 (P.4): Working groups staffed by member-company employees..", "rationale_leads": "Capped at 2 by mlcommons.07 (P.4): Working groups staffed by member-company employees.." }, "A": { "anchor": 0, "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "dimension": "A", "evaluator": "mlcommons", "signals": [ "mlcommons.03", "mlcommons.10" ], "value": 0, "mechanism": "lab-controlled", "graph_id": "assessment#718", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 0, "b": [ "mlcommons.10" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 0, "b": [ "mlcommons.10" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 0, "b": [ "mlcommons.10" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 0, "b": [ "mlcommons.10" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "mlcommons.03", "mlcommons.10" ] } }, "rationale_derived": "Capped at 0 by mlcommons.10 (A.1): The only documented engagement evaluated a publicly released model inside an enclave; no pre-release access from a lab is documented.. Floors up to 0 from mlcommons.03 do not exceed the cap.", "rationale_leads": "Capped at 0 by mlcommons.10 (A.1): The only documented engagement evaluated a publicly released model inside an enclave; no pre-release access from a lab is documented.. Floors up to 0 from mlcommons.03 do not exceed the cap." }, "S": { "anchor": 3, "assessed": "2026-09-14", "assessor": "yohei/claude v0", "dimension": "S", "evaluator": "mlcommons", "signals": [ "mlcommons.05" ], "value": 3, "graph_id": "assessment#719", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "mlcommons.05" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "mlcommons.05" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "mlcommons.05" ] }, "spans": { "v": 3, "b": [ "mlcommons.05" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "mlcommons.05" ] } }, "rationale_derived": "Floored at 3 by mlcommons.05 (S.5): Benchmark design set by working group.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by mlcommons.05 (S.5): Benchmark design set by working group.. No admissible signal caps it." }, "R": { "anchor": 2, "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "dimension": "R", "evaluator": "mlcommons", "signals": [ "mlcommons.06" ], "value": 2, "mechanism": "self-imposed", "graph_id": "assessment#720", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "mlcommons.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "mlcommons.06" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "mlcommons.06" ] }, "spans": { "v": 2, "b": [ "mlcommons.06" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "mlcommons.06" ] } }, "rationale_derived": "Floored at 2 by mlcommons.06 (R.7): Publishes results.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by mlcommons.06 (R.7): Publishes results.. No admissible signal caps it." }, "M": { "anchor": 3, "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "dimension": "M", "evaluator": "mlcommons", "signals": [ "mlcommons.02" ], "value": 3, "graph_id": "assessment#721", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "mlcommons.02" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "mlcommons.02" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "mlcommons.02" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "mlcommons.02" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "mlcommons.02" ] } }, "rationale_derived": "Floored at 3 by mlcommons.02 (M.1): Open benchmark specification and methodology.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by mlcommons.02 (M.1): Open benchmark specification and methodology.. No admissible signal caps it." }, "X": { "anchor": 3, "assessed": "2026-09-14", "assessor": "yohei/claude v0", "dimension": "X", "evaluator": "mlcommons", "signals": [ "mlcommons.08" ], "value": 3, "graph_id": "assessment#722", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "mlcommons.08" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "mlcommons.08" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "mlcommons.08" ] }, "spans": { "v": 3, "b": [ "mlcommons.08" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "mlcommons.08" ] } }, "rationale_derived": "Floored at 3 by mlcommons.08 (X.4): Benchmarks free to use.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by mlcommons.08 (X.4): Benchmarks free to use.. No admissible signal caps it." } }, "signals": [ { "id": "mlcommons.01", "evaluator": "mlcommons", "dimension": "F", "direction": "against", "claim": "Funded by member companies, including the labs whose models are benchmarked.", "sources": [ "mlcommons", "mlcommons-ailuminate", "mlcommons-siliconangle" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 1 }, "rule": "F.5", "quote": "MLCommons is an industry consortium backed by several dozen tech firms", "quote_source": "mlcommons-siliconangle", "graph_id": "signal#446", "source_graph_ids": [ "source#125", "source#120", "source#124" ] }, { "id": "mlcommons.02", "evaluator": "mlcommons", "dimension": "M", "direction": "for", "claim": "Open benchmark specification and methodology.", "sources": [ "mlcommons", "averi-pilot", "mlcommons-demo-dataset" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "M.1", "quote": "The Official Test dataset is private to avoid SUTs “gaming” the benchmark", "quote_source": "mlcommons-demo-dataset", "graph_id": "signal#447", "source_graph_ids": [ "source#125", "source#42", "source#121" ] }, { "id": "mlcommons.03", "evaluator": "mlcommons", "dimension": "A", "direction": "for", "claim": "Used as neutral ground in the AVERI double-blind pilot.", "sources": [ "averi-pilot" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 0 }, "rule": "A.1", "quote": "AVERI, OpenMined, and MLCommons were not able to see the model’s weights", "quote_source": "averi-pilot", "graph_id": "signal#448", "source_graph_ids": [ "source#42" ] }, { "id": "mlcommons.04", "evaluator": "mlcommons", "dimension": "G", "direction": "against", "claim": "Consortium governance by members.", "sources": [ "mlcommons", "mlcommons-ailuminate", "mlcommons-propublica" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "G.4", "quote": "MLCommons is supported by over 125 members and affiliates", "quote_source": "mlcommons", "graph_id": "signal#449", "source_graph_ids": [ "source#125", "source#120", "source#123" ] }, { "id": "mlcommons.05", "evaluator": "mlcommons", "dimension": "S", "direction": "for", "claim": "Benchmark design set by working group.", "sources": [ "mlcommons", "mlcommons-demo-dataset" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "S.5", "quote": "designed and developed by the MLCommons AI Risk and Reliability working group", "quote_source": "mlcommons-demo-dataset", "graph_id": "signal#450", "source_graph_ids": [ "source#125", "source#121" ] }, { "id": "mlcommons.06", "evaluator": "mlcommons", "dimension": "R", "direction": "for", "claim": "Publishes results.", "sources": [ "mlcommons", "mlcommons-demo-dataset" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "R.7", "quote": "Public reports of 13 systems-under-test (SUTS)", "quote_source": "mlcommons-demo-dataset", "graph_id": "signal#451", "source_graph_ids": [ "source#125", "source#121" ] }, { "id": "mlcommons.07", "evaluator": "mlcommons", "dimension": "P", "direction": "against", "claim": "Working groups staffed by member-company employees.", "sources": [ "mlcommons", "mlcommons-ailuminate" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "P.4", "quote": "People from the following organizations have collaborated within the working group", "quote_source": "mlcommons-ailuminate", "graph_id": "signal#452", "source_graph_ids": [ "source#125", "source#120" ] }, { "id": "mlcommons.08", "evaluator": "mlcommons", "dimension": "X", "direction": "for", "claim": "Benchmarks free to use.", "sources": [ "mlcommons", "mlcommons-demo-dataset" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "X.4", "quote": "publicly released the AILuminate DEMO prompt dataset, a collection of 1,200 prompts", "quote_source": "mlcommons-demo-dataset", "graph_id": "signal#453", "source_graph_ids": [ "source#125", "source#121" ] }, { "id": "mlcommons.09", "evaluator": "mlcommons", "dimension": "P", "direction": "against", "claim": "President is a Google senior staff engineer; members include the frontier developers whose models are benchmarked.", "sources": [ "evaluators-ledger", "mlcommons-leadership" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-11", "bound": { "cap": 2 }, "rule": "P.4", "quote": "Peter Mattson is a senior staff engineer at Google. He founded and is President of MLCommons", "quote_source": "mlcommons-leadership", "graph_id": "signal#454", "source_graph_ids": [ "source#81", "source#122" ] }, { "id": "mlcommons.10", "evaluator": "mlcommons", "dimension": "A", "direction": "against", "claim": "The only documented engagement evaluated a publicly released model inside an enclave; no pre-release access from a lab is documented.", "sources": [ "averi-pilot" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "as_of": "2026-09-15", "bound": { "cap": 0 }, "rule": "A.1", "quote": "AVERI, OpenMined, and MLCommons were not able to see the model’s weights", "quote_source": "averi-pilot", "graph_id": "signal#455", "source_graph_ids": [ "source#42" ] } ], "scores": { "lab": 43, "regulator": 44, "public": 46, "equal": 50 }, "scores_by_policy": { "leads": { "lab": { "score": 43, "band": "disqualifying", "coverage": 8, "range": [ 43, 43 ] }, "regulator": { "score": 44, "band": "disqualifying", "coverage": 8, "range": [ 44, 44 ] }, "public": { "score": 46, "band": "disqualifying", "coverage": 8, "range": [ 46, 46 ] }, "equal": { "score": 50, "band": "disqualifying", "coverage": 8, "range": [ 50, 50 ] } }, "standard": { "lab": { "score": 43, "band": "disqualifying", "coverage": 8, "range": [ 43, 43 ] }, "regulator": { "score": 44, "band": "disqualifying", "coverage": 8, "range": [ 44, 44 ] }, "public": { "score": 46, "band": "disqualifying", "coverage": 8, "range": [ 46, 46 ] }, "equal": { "score": 50, "band": "disqualifying", "coverage": 8, "range": [ 50, 50 ] } }, "against_interest": { "lab": { "score": 31, "band": "disqualifying", "coverage": 5, "range": [ 20, 55 ] }, "regulator": { "score": 33, "band": "disqualifying", "coverage": 5, "range": [ 21, 56 ] }, "public": { "score": 38, "band": "disqualifying", "coverage": 5, "range": [ 25, 60 ] }, "equal": { "score": 40, "band": "disqualifying", "coverage": 5, "range": [ 25, 63 ] } }, "spans": { "lab": { "score": 43, "band": "disqualifying", "coverage": 8, "range": [ 43, 43 ] }, "regulator": { "score": 44, "band": "disqualifying", "coverage": 8, "range": [ 44, 44 ] }, "public": { "score": 46, "band": "disqualifying", "coverage": 8, "range": [ 46, 46 ] }, "equal": { "score": 50, "band": "disqualifying", "coverage": 8, "range": [ 50, 50 ] } }, "primary": { "lab": { "score": 50, "band": "clear", "coverage": 1, "range": [ 4, 96 ] }, "regulator": { "score": 50, "band": "clear", "coverage": 1, "range": [ 5, 95 ] }, "public": { "score": 50, "band": "clear", "coverage": 1, "range": [ 8, 93 ] }, "equal": { "score": 50, "band": "clear", "coverage": 1, "range": [ 6, 94 ] } } }, "band": "disqualifying", "coverage": 8, "floor": "A", "what_moves": { "lab": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 48, "band": "disqualifying", "base": 43 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 45, "band": "disqualifying", "base": 43 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 46, "band": "disqualifying", "base": 43 }, { "dimension": "A", "from_value": 0, "to_value": 1, "score": 48, "band": "conditional", "base": 43 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 47, "band": "disqualifying", "base": 43 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 47, "band": "disqualifying", "base": 43 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 45, "band": "disqualifying", "base": 43 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 44, "band": "disqualifying", "base": 43 } ], "regulator": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 48, "band": "disqualifying", "base": 44 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 46, "band": "disqualifying", "base": 44 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 46, "band": "disqualifying", "base": 44 }, { "dimension": "A", "from_value": 0, "to_value": 1, "score": 49, "band": "conditional", "base": 44 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 49, "band": "disqualifying", "base": 44 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 48, "band": "disqualifying", "base": 44 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 46, "band": "disqualifying", "base": 44 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 44, "band": "disqualifying", "base": 44 } ], "public": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 53, "band": "disqualifying", "base": 46 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 50, "band": "disqualifying", "base": 46 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 50, "band": "disqualifying", "base": 46 }, { "dimension": "A", "from_value": 0, "to_value": 1, "score": 48, "band": "conditional", "base": 46 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 49, "band": "disqualifying", "base": 46 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 51, "band": "disqualifying", "base": 46 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 48, "band": "disqualifying", "base": 46 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 48, "band": "disqualifying", "base": 46 } ], "equal": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 53, "band": "disqualifying", "base": 50 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 53, "band": "disqualifying", "base": 50 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 53, "band": "disqualifying", "base": 50 }, { "dimension": "A", "from_value": 0, "to_value": 1, "score": 53, "band": "conditional", "base": 50 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 53, "band": "disqualifying", "base": 50 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 53, "band": "disqualifying", "base": 50 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 53, "band": "disqualifying", "base": 50 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 53, "band": "disqualifying", "base": 50 } ] } }, { "id": "msft", "name": "Microsoft AI Red Team", "type": "bigtech", "hq": "Redmond, US", "domains": [ "jailbreak", "cyber", "misuse" ], "confidence": "med", "summary": "External red team for other labs' models; not independent of Big Tech.", "what_would_move_the_score": "Not a candidate for an independence-critical role; useful as a supplementary red team.", "role": "lab-team", "dissent": { "lower": "Governance (1 to 0). The Microsoft AI Red Team is an internal function formed in 2018 inside a frontier developer, described on Microsoft's own Learn hub and Security Blog; G.6 and F.1 place a business unit of a developer at 0. Only the unaudited status of two pages that load and say exactly this holds the value at 1; both are the organization's own admissions.", "higher": "Publication rights (1 to 2). The team publishes under its own name: a whitepaper on lessons from red teaming more than 100 generative AI products, case studies, and the open-source PyRIT toolkit. R.3's second clause caps an organization that publishes its own reports on other work at 2, not 1, even where findings on other labs' models stay inside system cards." }, "list_group": "commercial", "graph_id": "evaluator#220", "values": { "F": 1, "G": 0, "P": 0, "A": 3, "S": 2, "R": 2, "M": 3, "X": 1 }, "values_by_policy": { "leads": { "F": 1, "G": 0, "P": 0, "A": 3, "S": 2, "R": 2, "M": 3, "X": 1 }, "standard": { "F": null, "G": 0, "P": 0, "A": 3, "S": 2, "R": 2, "M": 3, "X": 1 }, "against_interest": { "F": null, "G": 0, "P": 0, "A": null, "S": 2, "R": 2, "M": null, "X": 1 }, "spans": { "F": null, "G": 0, "P": 0, "A": 3, "S": null, "R": 2, "M": 3, "X": 1 }, "primary": { "F": null, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "msft", "dimension": "F", "value": 1, "anchor": 1, "signals": [ "msft.01" ], "rationale": "Anchor 0 on funding given the cited signals. Evidence-limited: the cited sources are imported or unaudited only, so the value is held at 1 (the reading on the record would be 0) until a live source is confirmed.", "assessed": "2026-09-15", "assessor": "yohei/claude v0-final", "evidence_limited": true, "graph_id": "assessment#723", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 1, "b": [ "msft.01" ], "h": true, "c": false, "u": false, "x": [] }, "standard": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "msft.01" ] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "msft.01" ] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "msft.01" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "msft.01" ] } }, "rationale_derived": "Unevidenced under this policy: no admissible signal sets a bound. Not counted: msft.01 (no confirmed source (unaudited)).", "rationale_leads": "Capped at 1 by msft.01 (F.1): Microsoft is a major OpenAI investor and a frontier developer with its own CAISI agreement.. Held: C14: sources not all confirmed." }, "G": { "evaluator": "msft", "dimension": "G", "value": 0, "anchor": 0, "signals": [ "msft.02", "msft.09" ], "rationale": "Anchor 0 on governance given the cited signals. Evidence-limited: the cited sources are imported or unaudited only, so the value is held at 1 (the reading on the record would be 0) until a live source is confirmed.", "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#724", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 0, "b": [ "msft.02", "msft.09" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 0, "b": [ "msft.02", "msft.09" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 0, "b": [ "msft.02", "msft.09" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 0, "b": [ "msft.02", "msft.09" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "msft.02", "msft.09" ] } }, "rationale_derived": "Capped at 0 by msft.02 (G.6): Business unit of a frontier developer.; capped at 0 by msft.09 (G.6): Microsoft's AI red team has operated since 2018 as an internal function and reports having red-teamed more than 100 generative AI products; it is separate from product teams, not from the developer..", "rationale_leads": "Capped at 0 by msft.02 (G.6): Business unit of a frontier developer.; capped at 0 by msft.09 (G.6): Microsoft's AI red team has operated since 2018 as an internal function and reports having red-teamed more than 100 generative AI products; it is separate from product teams, not from the developer.." }, "P": { "evaluator": "msft", "dimension": "P", "value": 0, "anchor": 0, "signals": [ "msft.06" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#725", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 0, "b": [ "msft.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 0, "b": [ "msft.06" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 0, "b": [ "msft.06" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 0, "b": [ "msft.06" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "msft.06" ] } }, "rationale_derived": "Capped at 0 by msft.06 (P.7): Employees of a lab..", "rationale_leads": "Capped at 0 by msft.06 (P.7): Employees of a lab.." }, "A": { "evaluator": "msft", "dimension": "A", "value": 3, "anchor": 3, "signals": [ "msft.05" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "mechanism": "lab-controlled", "evidence_limited": true, "graph_id": "assessment#726", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "msft.05" ], "h": true, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "msft.05" ], "h": true, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "msft.05" ] }, "spans": { "v": 3, "b": [ "msft.05" ], "h": true, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "msft.05" ] } }, "rationale_derived": "Floored at 3 by msft.05 (A.5): Deep access when engaged.. No admissible signal caps it. Held: C16: a 4 needs a tier-1/2 source or two independent sources, one not self-published.", "rationale_leads": "Floored at 3 by msft.05 (A.5): Deep access when engaged.. No admissible signal caps it. Held: C16: a 4 needs a tier-1/2 source or two independent sources, one not self-published." }, "S": { "evaluator": "msft", "dimension": "S", "value": 2, "anchor": 2, "signals": [ "msft.07" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#727", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "msft.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "msft.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "msft.07" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "msft.07" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "msft.07" ] } }, "rationale_derived": "Capped at 2 by msft.07 (S.2): Scope set by engagement..", "rationale_leads": "Capped at 2 by msft.07 (S.2): Scope set by engagement.." }, "R": { "evaluator": "msft", "dimension": "R", "value": 2, "anchor": 2, "signals": [ "msft.03" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "lab-controlled", "graph_id": "assessment#728", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "msft.03" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "msft.03" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "msft.03" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "msft.03" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "msft.03" ] } }, "rationale_derived": "Capped at 2 by msft.03 (R.3): Findings for other labs are rarely published under the team's own name..", "rationale_leads": "Capped at 2 by msft.03 (R.3): Findings for other labs are rarely published under the team's own name.." }, "M": { "evaluator": "msft", "dimension": "M", "value": 3, "anchor": 3, "signals": [ "msft.04" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#729", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "msft.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "msft.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "msft.04" ] }, "spans": { "v": 3, "b": [ "msft.04" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "msft.04" ] } }, "rationale_derived": "Floored at 3 by msft.04 (M.1): Publishes methodology (PyRIT) openly.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by msft.04 (M.1): Publishes methodology (PyRIT) openly.. No admissible signal caps it." }, "X": { "evaluator": "msft", "dimension": "X", "value": 1, "anchor": 1, "signals": [ "msft.08" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#730", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 1, "b": [ "msft.08" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "msft.08" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "msft.08" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "msft.08" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "msft.08" ] } }, "rationale_derived": "Capped at 1 by msft.08 (X.2): Sells AI products..", "rationale_leads": "Capped at 1 by msft.08 (X.2): Sells AI products.." } }, "signals": [ { "id": "msft.01", "evaluator": "msft", "dimension": "F", "direction": "against", "claim": "Microsoft is a major OpenAI investor and a frontier developer with its own CAISI agreement.", "sources": [ "csa-caisi" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 0 }, "rule": "F.1", "quote": "Google (DeepMind), Microsoft, and xAI signed agreements with the US Center for AI Standards and Innovation (CAISI)", "quote_source": "csa-caisi", "graph_id": "signal#456", "source_graph_ids": [ "source#57" ] }, { "id": "msft.02", "evaluator": "msft", "dimension": "G", "direction": "against", "claim": "Business unit of a frontier developer.", "sources": [ "ms-redteam" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 0 }, "rule": "G.6", "quote": "guidance and best practices from the industry leading Microsoft AI Red Team", "quote_source": "ms-redteam", "graph_id": "signal#457", "source_graph_ids": [ "source#127" ] }, { "id": "msft.03", "evaluator": "msft", "dimension": "R", "direction": "against", "claim": "Findings for other labs are rarely published under the team's own name.", "sources": [ "ms-redteam", "ms-redteam-100" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "R.3", "quote": "Microsoft’s AI red team is excited to share our whitepaper", "quote_source": "ms-redteam-100", "graph_id": "signal#458", "source_graph_ids": [ "source#127", "source#126" ] }, { "id": "msft.04", "evaluator": "msft", "dimension": "M", "direction": "for", "claim": "Publishes methodology (PyRIT) openly.", "sources": [ "ms-redteam" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "M.1", "quote": "Microsoft's Open Automation Framework to Red Team Generative AI Systems (PyRIT)", "quote_source": "ms-redteam", "graph_id": "signal#459", "source_graph_ids": [ "source#127" ] }, { "id": "msft.05", "evaluator": "msft", "dimension": "A", "direction": "for", "claim": "Deep access when engaged.", "sources": [ "ms-redteam", "ms-redteam-100" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 4 }, "rule": "A.5", "quote": "red teaming has become a key part of Microsoft’s approach to generative AI product development", "quote_source": "ms-redteam-100", "graph_id": "signal#460", "source_graph_ids": [ "source#127", "source#126" ] }, { "id": "msft.06", "evaluator": "msft", "dimension": "P", "direction": "against", "claim": "Employees of a lab.", "sources": [ "ms-redteam" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 0 }, "rule": "P.7", "quote": "guidance and best practices from the industry leading Microsoft AI Red Team", "quote_source": "ms-redteam", "graph_id": "signal#461", "source_graph_ids": [ "source#127" ] }, { "id": "msft.07", "evaluator": "msft", "dimension": "S", "direction": "against", "claim": "Scope set by engagement.", "sources": [ "ms-redteam" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "S.2", "graph_id": "signal#462", "source_graph_ids": [ "source#127" ] }, { "id": "msft.08", "evaluator": "msft", "dimension": "X", "direction": "against", "claim": "Sells AI products.", "sources": [ "ms-redteam" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 1 }, "rule": "X.2", "quote": "What is Azure AI Content Safety?", "quote_source": "ms-redteam", "graph_id": "signal#463", "source_graph_ids": [ "source#127" ] }, { "id": "msft.09", "evaluator": "msft", "dimension": "G", "direction": "against", "claim": "Microsoft's AI red team has operated since 2018 as an internal function and reports having red-teamed more than 100 generative AI products; it is separate from product teams, not from the developer.", "sources": [ "ms-redteam", "ms-redteam-100" ], "recorded": "2026-09-15", "curator": "yohei/claude v0-final", "as_of": "2025-01-13", "bound": { "cap": 0 }, "rule": "G.6", "quote": "The AI red team was formed in 2018 to address the growing landscape of AI safety and security risks", "quote_source": "ms-redteam-100", "graph_id": "signal#464", "source_graph_ids": [ "source#127", "source#126" ] } ], "scores": { "lab": 46, "regulator": 47, "public": 32, "equal": 39 }, "scores_by_policy": { "leads": { "lab": { "score": 43, "band": "disqualifying", "coverage": 8, "range": [ 43, 43 ] }, "regulator": { "score": 44, "band": "disqualifying", "coverage": 8, "range": [ 44, 44 ] }, "public": { "score": 30, "band": "disqualifying", "coverage": 8, "range": [ 30, 30 ] }, "equal": { "score": 38, "band": "disqualifying", "coverage": 8, "range": [ 38, 38 ] } }, "standard": { "lab": { "score": 46, "band": "disqualifying", "coverage": 7, "range": [ 38, 56 ] }, "regulator": { "score": 47, "band": "disqualifying", "coverage": 7, "range": [ 40, 55 ] }, "public": { "score": 32, "band": "disqualifying", "coverage": 7, "range": [ 24, 49 ] }, "equal": { "score": 39, "band": "disqualifying", "coverage": 7, "range": [ 34, 47 ] } }, "against_interest": { "lab": { "score": 31, "band": "disqualifying", "coverage": 5, "range": [ 16, 63 ] }, "regulator": { "score": 32, "band": "disqualifying", "coverage": 5, "range": [ 18, 63 ] }, "public": { "score": 25, "band": "disqualifying", "coverage": 5, "range": [ 16, 51 ] }, "equal": { "score": 25, "band": "disqualifying", "coverage": 5, "range": [ 16, 53 ] } }, "spans": { "lab": { "score": 45, "band": "disqualifying", "coverage": 6, "range": [ 30, 64 ] }, "regulator": { "score": 46, "band": "disqualifying", "coverage": 6, "range": [ 30, 65 ] }, "public": { "score": 29, "band": "disqualifying", "coverage": 6, "range": [ 19, 54 ] }, "equal": { "score": 38, "band": "disqualifying", "coverage": 6, "range": [ 28, 53 ] } }, "primary": { "lab": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "regulator": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "public": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "equal": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] } } }, "band": "disqualifying", "coverage": 7, "floor": "G", "what_moves": { "lab": [ { "dimension": "G", "from_value": 0, "to_value": 1, "score": 49, "band": "disqualifying", "base": 46 }, { "dimension": "P", "from_value": 0, "to_value": 1, "score": 49, "band": "disqualifying", "base": 46 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 52, "band": "disqualifying", "base": 46 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 51, "band": "disqualifying", "base": 46 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 51, "band": "disqualifying", "base": 46 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 49, "band": "disqualifying", "base": 46 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 48, "band": "disqualifying", "base": 46 } ], "regulator": [ { "dimension": "G", "from_value": 0, "to_value": 1, "score": 50, "band": "disqualifying", "base": 47 }, { "dimension": "P", "from_value": 0, "to_value": 1, "score": 50, "band": "disqualifying", "base": 47 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 53, "band": "disqualifying", "base": 47 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 53, "band": "disqualifying", "base": 47 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 51, "band": "disqualifying", "base": 47 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 50, "band": "disqualifying", "base": 47 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 47, "band": "disqualifying", "base": 47 } ], "public": [ { "dimension": "G", "from_value": 0, "to_value": 1, "score": 37, "band": "disqualifying", "base": 32 }, { "dimension": "P", "from_value": 0, "to_value": 1, "score": 37, "band": "disqualifying", "base": 32 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 33, "band": "disqualifying", "base": 32 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 35, "band": "disqualifying", "base": 32 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 38, "band": "disqualifying", "base": 32 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 33, "band": "disqualifying", "base": 32 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 33, "band": "disqualifying", "base": 32 } ], "equal": [ { "dimension": "G", "from_value": 0, "to_value": 1, "score": 43, "band": "disqualifying", "base": 39 }, { "dimension": "P", "from_value": 0, "to_value": 1, "score": 43, "band": "disqualifying", "base": 39 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 43, "band": "disqualifying", "base": 39 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 43, "band": "disqualifying", "base": 39 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 43, "band": "disqualifying", "base": 39 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 43, "band": "disqualifying", "base": 39 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 43, "band": "disqualifying", "base": 39 } ] } }, { "confidence": "low", "domains": [ "bio", "misuse", "assurance" ], "hq": "Albany, US", "id": "nemesys", "name": "Nemesys Insights", "role": "vendor", "summary": "Red-teaming firm led by the director of a university red-teaming center; paid assessor on Amazon's Nova model papers; subcontractor on the EU Lot 1 CBRN consortium. Paid contracting work for labs and governments makes this a vendor, not an independent referee.", "type": "private", "what_would_move_the_score": "Funding disclosure and a published policy on lab clients.", "dissent": { "lower": "Publication (R) at 1 could be 0. The public record shows no Nemesys-authored report on any model; every finding appears inside Amazon's own technical reports, which Amazon wrote, framed and released. Amazon \"shares input prompts and corresponding model outputs\" with the assessor and asks for a verdict against its own thresholds. Where the lab both commissions the work and holds the only pen, anchor 0 (lab approval required) is the closer description.", "higher": "Publication (R) at 1 could be 2. Amazon's reports name Nemesys, describe its 120-prompt threshold analysis and its 800-participant uplift study, and print its conclusion; the assessor's method and verdict reached the public in the lab's paper rather than as a bare citation. That is nearer to a published finding with lab review than to lab-edited summaries only." }, "list_group": "commercial", "graph_id": "evaluator#221", "values": { "F": 1, "G": 1, "P": 2, "A": 1, "S": 1, "R": 1, "M": 2, "X": 1 }, "values_by_policy": { "leads": { "F": 1, "G": 1, "P": 2, "A": 1, "S": 1, "R": 1, "M": 2, "X": 1 }, "standard": { "F": 1, "G": 1, "P": 2, "A": 1, "S": 1, "R": 1, "M": 2, "X": 1 }, "against_interest": { "F": 1, "G": 1, "P": 2, "A": 1, "S": 1, "R": 1, "M": 2, "X": 1 }, "spans": { "F": 1, "G": 1, "P": 2, "A": 1, "S": 1, "R": 1, "M": 2, "X": 1 }, "primary": { "F": 2, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "nemesys", "dimension": "F", "value": 1, "anchor": 1, "signals": [ "nemesys.01", "nemesys.02" ], "rationale": "Anchor 1 from the cited rows; low confidence, imported evidence.", "assessed": "2026-09-14", "assessor": "yohei/claude v0.4", "resolution": { "rule": "F.5", "value": 1, "note": "Lab revenue is the only revenue disclosed; the public subcontract is real but its share is not public. F.5 decides: 1." }, "graph_id": "assessment#731", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 1, "b": [ "nemesys.01" ], "h": false, "c": true, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "nemesys.01" ], "h": false, "c": true, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "nemesys.01" ], "h": false, "c": true, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "nemesys.01" ], "h": false, "c": true, "u": false, "x": [] }, "primary": { "v": 2, "b": [ "nemesys.02" ], "h": false, "c": false, "u": false, "x": [ "nemesys.01" ] } }, "rationale_derived": "Conflict: floor 2 from nemesys.02 (F.3) against cap 1 from nemesys.01 (F.5); resolved at 1 by F.5: Lab revenue is the only revenue disclosed; the public subcontract is real but its share is not public. F.5 decides: 1.", "rationale_leads": "Conflict: floor 2 from nemesys.02 (F.3) against cap 1 from nemesys.01 (F.5); resolved at 1 by F.5: Lab revenue is the only revenue disclosed; the public subcontract is real but its share is not public. F.5 decides: 1." }, "G": { "evaluator": "nemesys", "dimension": "G", "value": 1, "anchor": 1, "signals": [ "nemesys.03" ], "rationale": "Anchor 1 from the cited rows; low confidence, imported evidence. Evidence-limited: the cited sources are imported or unaudited only, so the value is held at 2 (the reading on the record would be 1) until a live source is confirmed.", "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#732", "evidence_tier": "tier 2 (ledger)", "derived": { "leads": { "v": 1, "b": [ "nemesys.03" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "nemesys.03" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "nemesys.03" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "nemesys.03" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "nemesys.03" ] } }, "rationale_derived": "Capped at 1 by nemesys.03 (G.2): No policy published..", "rationale_leads": "Capped at 1 by nemesys.03 (G.2): No policy published.." }, "P": { "evaluator": "nemesys", "dimension": "P", "value": 2, "anchor": 2, "signals": [ "nemesys.04" ], "rationale": "Anchor 2 from the cited rows; low confidence, imported evidence.", "assessed": "2026-09-14", "assessor": "yohei/claude v0.4", "graph_id": "assessment#733", "evidence_tier": "tier 2 (ledger)", "derived": { "leads": { "v": 2, "b": [ "nemesys.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "nemesys.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "nemesys.04" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "nemesys.04" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "nemesys.04" ] } }, "rationale_derived": "Capped at 2 by nemesys.04 (P.6): Leadership from a university red-team center; lab ties not established..", "rationale_leads": "Capped at 2 by nemesys.04 (P.6): Leadership from a university red-team center; lab ties not established.." }, "A": { "evaluator": "nemesys", "dimension": "A", "value": 1, "anchor": 1, "signals": [ "nemesys.05" ], "rationale": "Anchor 2 from the cited rows; low confidence, imported evidence.", "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "lab-controlled", "graph_id": "assessment#734", "evidence_tier": "tier 2 (ledger)", "derived": { "leads": { "v": 1, "b": [ "nemesys.05" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "nemesys.05" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "nemesys.05" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "nemesys.05" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "nemesys.05" ] } }, "rationale_derived": "Capped at 1 by nemesys.05 (A.1): Access through client engagement..", "rationale_leads": "Capped at 1 by nemesys.05 (A.1): Access through client engagement.." }, "S": { "evaluator": "nemesys", "dimension": "S", "value": 1, "anchor": 1, "signals": [ "nemesys.06" ], "rationale": "Anchor 1 from the cited rows; low confidence, imported evidence. Evidence-limited: the cited sources are imported or unaudited only, so the value is held at 2 (the reading on the record would be 1) until a live source is confirmed.", "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#735", "evidence_tier": "tier 2 (ledger)", "derived": { "leads": { "v": 1, "b": [ "nemesys.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "nemesys.06" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "nemesys.06" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "nemesys.06" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "nemesys.06" ] } }, "rationale_derived": "Capped at 1 by nemesys.06 (S.1): Scope set by the lab client..", "rationale_leads": "Capped at 1 by nemesys.06 (S.1): Scope set by the lab client.." }, "R": { "evaluator": "nemesys", "dimension": "R", "value": 1, "anchor": 1, "signals": [ "nemesys.07" ], "rationale": "Anchor 1 from the cited rows; low confidence, imported evidence. Evidence-limited: the cited sources are imported or unaudited only, so the value is held at 2 (the reading on the record would be 1) until a live source is confirmed.", "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "lab-controlled", "graph_id": "assessment#736", "evidence_tier": "tier 2 (ledger)", "derived": { "leads": { "v": 1, "b": [ "nemesys.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "nemesys.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "nemesys.07" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "nemesys.07" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "nemesys.07" ] } }, "rationale_derived": "Capped at 1 by nemesys.07 (R.3): Findings appear inside the lab's papers..", "rationale_leads": "Capped at 1 by nemesys.07 (R.3): Findings appear inside the lab's papers.." }, "M": { "evaluator": "nemesys", "dimension": "M", "value": 2, "anchor": 2, "signals": [ "nemesys.08" ], "rationale": "Anchor 2 from the cited rows; low confidence, imported evidence.", "assessed": "2026-09-14", "assessor": "yohei/claude v0.4", "graph_id": "assessment#737", "evidence_tier": "tier 2 (ledger)", "derived": { "leads": { "v": 2, "b": [ "nemesys.08" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "nemesys.08" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "nemesys.08" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "nemesys.08" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "nemesys.08" ] } }, "rationale_derived": "Capped at 2 by nemesys.08 (M.2): Methods described in the lab's papers only..", "rationale_leads": "Capped at 2 by nemesys.08 (M.2): Methods described in the lab's papers only.." }, "X": { "evaluator": "nemesys", "dimension": "X", "value": 1, "anchor": 1, "signals": [ "nemesys.09" ], "rationale": "Anchor 2 from the cited rows; low confidence, imported evidence.", "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#738", "evidence_tier": "tier 2 (ledger)", "derived": { "leads": { "v": 1, "b": [ "nemesys.09" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "nemesys.09" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "nemesys.09" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "nemesys.09" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "nemesys.09" ] } }, "rationale_derived": "Capped at 1 by nemesys.09 (X.2): Consults for a lab..", "rationale_leads": "Capped at 1 by nemesys.09 (X.2): Consults for a lab.." } }, "signals": [ { "id": "nemesys.01", "evaluator": "nemesys", "dimension": "F", "direction": "against", "claim": "Paid assessor for Amazon per the Nova papers; private LLC with no funding disclosed.", "sources": [ "evaluators-ledger", "nemesys-nova-html" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 1 }, "rule": "F.5", "quote": "We share input prompts and corresponding model outputs with third-party assessors (e.g., Nemesys, METR)", "quote_source": "nemesys-nova-html", "graph_id": "signal#465", "source_graph_ids": [ "source#81", "source#129" ] }, { "id": "nemesys.02", "evaluator": "nemesys", "dimension": "F", "direction": "for", "claim": "EU Lot 1 subcontractor under the FAR.AI consortium.", "sources": [ "ted-864574", "nemesys-far-blog" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "floor": 2 }, "rule": "F.3", "quote": "Nemesys Insights (chemical, biological, radiological, and nuclear threats)", "quote_source": "nemesys-far-blog", "graph_id": "signal#466", "source_graph_ids": [ "source#184", "source#128" ] }, { "id": "nemesys.03", "evaluator": "nemesys", "dimension": "G", "direction": "against", "claim": "No policy published.", "sources": [ "evaluators-ledger", "nemesys-site" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 1 }, "rule": "G.2", "quote": "Nemesys Insights, LLC is a strategic analysis and advisory company", "quote_source": "nemesys-site", "graph_id": "signal#467", "source_graph_ids": [ "source#81", "source#133" ] }, { "id": "nemesys.04", "evaluator": "nemesys", "dimension": "P", "direction": "against", "claim": "Leadership from a university red-team center; lab ties not established.", "sources": [ "evaluators-ledger", "nemesys-ourstory", "nemesys-ourteam" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 2 }, "rule": "P.6", "quote": "promote the tools and techniques developed by the Center for Advanced Red Teaming (CART)", "quote_source": "nemesys-ourstory", "graph_id": "signal#468", "source_graph_ids": [ "source#81", "source#131", "source#132" ] }, { "id": "nemesys.05", "evaluator": "nemesys", "dimension": "A", "direction": "against", "claim": "Access through client engagement.", "sources": [ "evaluators-ledger", "nemesys-nova-html" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 1 }, "rule": "A.1", "quote": "Nemesys Insights executed an Exploratory Critical-Capability Threshold Analysis", "quote_source": "nemesys-nova-html", "graph_id": "signal#469", "source_graph_ids": [ "source#81", "source#129" ] }, { "id": "nemesys.06", "evaluator": "nemesys", "dimension": "S", "direction": "against", "claim": "Scope set by the lab client.", "sources": [ "evaluators-ledger", "nemesys-nova-html" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 1 }, "rule": "S.1", "quote": "their final assessment of whether the model is safe and remains within the defined thresholds for public release", "quote_source": "nemesys-nova-html", "graph_id": "signal#470", "source_graph_ids": [ "source#81", "source#129" ] }, { "id": "nemesys.07", "evaluator": "nemesys", "dimension": "R", "direction": "against", "claim": "Findings appear inside the lab's papers.", "sources": [ "evaluators-ledger", "nemesys-nova-html" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 1 }, "rule": "R.3", "quote": "reviewed results on test sets, scoring rubrics, and safety-judge notes to verify our internal findings", "quote_source": "nemesys-nova-html", "graph_id": "signal#471", "source_graph_ids": [ "source#81", "source#129" ] }, { "id": "nemesys.08", "evaluator": "nemesys", "dimension": "M", "direction": "against", "claim": "Methods described in the lab's papers only.", "sources": [ "evaluators-ledger", "nemesys-nova-html" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 2 }, "rule": "M.2", "quote": "They selected 120 \"uplift indicator\" prompts—60 in synthetic biology and 60 in microbiology/delivery systems", "quote_source": "nemesys-nova-html", "graph_id": "signal#472", "source_graph_ids": [ "source#81", "source#129" ] }, { "id": "nemesys.09", "evaluator": "nemesys", "dimension": "X", "direction": "against", "claim": "Consults for a lab.", "sources": [ "evaluators-ledger", "nemesys-nova2-html" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 1 }, "rule": "X.2", "quote": "we worked with Nemesys Insights to conduct a large-scale, independent uplift study", "quote_source": "nemesys-nova2-html", "graph_id": "signal#473", "source_graph_ids": [ "source#81", "source#130" ] } ], "scores": { "lab": 30, "regulator": 30, "public": 30, "equal": 31 }, "scores_by_policy": { "leads": { "lab": { "score": 30, "band": "conditional", "coverage": 8, "range": [ 30, 30 ] }, "regulator": { "score": 30, "band": "conditional", "coverage": 8, "range": [ 30, 30 ] }, "public": { "score": 30, "band": "conditional", "coverage": 8, "range": [ 30, 30 ] }, "equal": { "score": 31, "band": "conditional", "coverage": 8, "range": [ 31, 31 ] } }, "standard": { "lab": { "score": 30, "band": "conditional", "coverage": 8, "range": [ 30, 30 ] }, "regulator": { "score": 30, "band": "conditional", "coverage": 8, "range": [ 30, 30 ] }, "public": { "score": 30, "band": "conditional", "coverage": 8, "range": [ 30, 30 ] }, "equal": { "score": 31, "band": "conditional", "coverage": 8, "range": [ 31, 31 ] } }, "against_interest": { "lab": { "score": 30, "band": "conditional", "coverage": 8, "range": [ 30, 30 ] }, "regulator": { "score": 30, "band": "conditional", "coverage": 8, "range": [ 30, 30 ] }, "public": { "score": 30, "band": "conditional", "coverage": 8, "range": [ 30, 30 ] }, "equal": { "score": 31, "band": "conditional", "coverage": 8, "range": [ 31, 31 ] } }, "spans": { "lab": { "score": 30, "band": "conditional", "coverage": 8, "range": [ 30, 30 ] }, "regulator": { "score": 30, "band": "conditional", "coverage": 8, "range": [ 30, 30 ] }, "public": { "score": 30, "band": "conditional", "coverage": 8, "range": [ 30, 30 ] }, "equal": { "score": 31, "band": "conditional", "coverage": 8, "range": [ 31, 31 ] } }, "primary": { "lab": { "score": 50, "band": "clear", "coverage": 1, "range": [ 9, 91 ] }, "regulator": { "score": 50, "band": "clear", "coverage": 1, "range": [ 8, 93 ] }, "public": { "score": 50, "band": "clear", "coverage": 1, "range": [ 13, 88 ] }, "equal": { "score": 50, "band": "clear", "coverage": 1, "range": [ 6, 94 ] } } }, "band": "conditional", "coverage": 8, "floor": "F", "what_moves": { "lab": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 34, "band": "conditional", "base": 30 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 32, "band": "conditional", "base": 30 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 32, "band": "conditional", "base": 30 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 35, "band": "conditional", "base": 30 }, { "dimension": "S", "from_value": 1, "to_value": 2, "score": 34, "band": "conditional", "base": 30 }, { "dimension": "R", "from_value": 1, "to_value": 2, "score": 33, "band": "conditional", "base": 30 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 32, "band": "conditional", "base": 30 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 31, "band": "conditional", "base": 30 } ], "regulator": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 34, "band": "conditional", "base": 30 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 33, "band": "conditional", "base": 30 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 33, "band": "conditional", "base": 30 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 35, "band": "conditional", "base": 30 }, { "dimension": "S", "from_value": 1, "to_value": 2, "score": 35, "band": "conditional", "base": 30 }, { "dimension": "R", "from_value": 1, "to_value": 2, "score": 34, "band": "conditional", "base": 30 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 33, "band": "conditional", "base": 30 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 30, "band": "conditional", "base": 30 } ], "public": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 36, "band": "conditional", "base": 30 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 34, "band": "conditional", "base": 30 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 34, "band": "conditional", "base": 30 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 31, "band": "conditional", "base": 30 }, { "dimension": "S", "from_value": 1, "to_value": 2, "score": 33, "band": "conditional", "base": 30 }, { "dimension": "R", "from_value": 1, "to_value": 2, "score": 35, "band": "conditional", "base": 30 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 31, "band": "conditional", "base": 30 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 31, "band": "conditional", "base": 30 } ], "equal": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 34, "band": "conditional", "base": 31 }, { "dimension": "G", "from_value": 1, "to_value": 2, "score": 34, "band": "conditional", "base": 31 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 34, "band": "conditional", "base": 31 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 34, "band": "conditional", "base": 31 }, { "dimension": "S", "from_value": 1, "to_value": 2, "score": 34, "band": "conditional", "base": 31 }, { "dimension": "R", "from_value": 1, "to_value": 2, "score": 34, "band": "conditional", "base": 31 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 34, "band": "conditional", "base": 31 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 34, "band": "conditional", "base": 31 } ] } }, { "id": "palisade", "name": "Palisade Research", "type": "nonprofit", "hq": "Berkeley, US", "domains": [ "cyber", "autonomy", "scheming" ], "confidence": "med", "summary": "Independent capability-elicitation research on released models: shutdown resistance, self-replication, offensive cyber.", "what_would_move_the_score": "Pre-release access on published terms would raise the access score without touching the others.", "role": "referee", "dissent": { "lower": "Access should be 0. Every published Palisade study (shutdown resistance, specification gaming, self-replication, Badllama) ran on released models through public APIs or open weights; no system card names Palisade as a pre-release tester and no engagement with a lab is on record. 'Little' pre-deployment access, with none documented, is anchor 0, and the 1 rests only on the claim's wording.", "higher": "Access should be 2. Palisade's work is cited by Anthropic's CEO and briefed to Congress, and labs have responded to its findings; if any developer supplied early checkpoints or extended API windows for the shutdown or hacking studies, that is anchor 2. The record says 'little', not none, and a single documented pre-release run with extended time would move it." }, "list_group": "referee", "graph_id": "evaluator#222", "values": { "F": 3, "G": 2, "P": 2, "A": 1, "S": 3, "R": 4, "M": 3, "X": 3 }, "values_by_policy": { "leads": { "F": 3, "G": 2, "P": 2, "A": 1, "S": 3, "R": 4, "M": 3, "X": 3 }, "standard": { "F": 3, "G": 2, "P": 2, "A": 1, "S": 3, "R": 4, "M": 3, "X": 3 }, "against_interest": { "F": 3, "G": null, "P": 2, "A": 1, "S": null, "R": 4, "M": null, "X": 3 }, "spans": { "F": 3, "G": 2, "P": 2, "A": null, "S": null, "R": 4, "M": 3, "X": 3 }, "primary": { "F": 3, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": 3 } }, "assessments": { "F": { "evaluator": "palisade", "dimension": "F", "value": 3, "anchor": 3, "signals": [ "palisade.01", "palisade.09", "palisade.11", "palisade.13" ], "rationale": "Anchor 3: philanthropic, no lab contracts found, but the main traced funder is two steps from a lab investor and no primary-filing negative is on file.", "assessed": "2026-09-15", "assessor": "yohei/claude v1.0", "open_questions": [ "Request to Palisade Research: the Form 990 Schedule B or a donor list naming contributors above $5,000 for 2024 and 2025, and any policy on accepting money from AI developers.", "Request to Coefficient Giving: the grant pages for the two Palisade Research general-support grants ($1,680,000 June 2024; $2,123,463 May 2025) so the $2,123,463 figure can be verified against the superseded $3,803,463 index total." ], "graph_id": "assessment#739", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 3, "b": [ "palisade.09", "palisade.11", "palisade.13" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "palisade.09", "palisade.13" ], "h": false, "c": false, "u": false, "x": [ "palisade.11" ] }, "against_interest": { "v": 3, "b": [ "palisade.09", "palisade.13" ], "h": false, "c": false, "u": false, "x": [ "palisade.11" ] }, "spans": { "v": 3, "b": [ "palisade.13" ], "h": false, "c": false, "u": false, "x": [ "palisade.09", "palisade.11" ] }, "primary": { "v": 3, "b": [ "palisade.01" ], "h": false, "c": false, "u": false, "x": [ "palisade.09", "palisade.11", "palisade.13" ] } }, "rationale_derived": "Capped at 3 by palisade.09 (F.6): About $3.8M in cumulative Coefficient Giving grants per the index snapshot; Coefficient's principal is an Anthropic Series A investor and board observer by his own account. No bounded negative on lab money is on file.; capped at 3 by palisade.13 (F.6): SFF recommended $1,133,000 to Palisade in 2025 (plus a $928,000 matching pledge); SFF's funder Jaan Tallinn led Anthropic's Series A.. Floors up to 3 from palisade.01 do not exceed the cap. Not counted under this policy: palisade.11 (no confirmed source (imported, unaudited)).", "rationale_leads": "Capped at 3 by palisade.09 (F.6): About $3.8M in cumulative Coefficient Giving grants per the index snapshot; Coefficient's principal is an Anthropic Series A investor and board observer by his own account. No bounded negative on lab money is on file.; capped at 3 by palisade.11 (F.6): Coefficient Giving's database records two grants totaling $2,123,463 for general support; the imported index total of $3,803,463 is marked differs pending reconciliation.; capped at 3 by palisade.13 (F.6): SFF recommended $1,133,000 to Palisade in 2025 (plus a $928,000 matching pledge); SFF's funder Jaan Tallinn led Anthropic's Series A.. Floors up to 3 from palisade.01 do not exceed the cap." }, "G": { "evaluator": "palisade", "dimension": "G", "value": 2, "anchor": 2, "signals": [ "palisade.06" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#740", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "palisade.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "palisade.06" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "palisade.06" ] }, "spans": { "v": 2, "b": [ "palisade.06" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "palisade.06" ] } }, "rationale_derived": "Floored at 2 by palisade.06 (G.1): Nonprofit.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by palisade.06 (G.1): Nonprofit.. No admissible signal caps it." }, "P": { "evaluator": "palisade", "dimension": "P", "value": 2, "anchor": 2, "signals": [ "palisade.07", "palisade.10" ], "rationale": "Anchor 3 (provisional): no lab board roles found (palisade.07), but the founder and executive director previously worked on Anthropic’s security team (palisade.10) — a former, disclosed lab tie, not a current one. Provisional readings are open to public correction through the contribution path.", "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "open_questions": [ "Request to Palisade Research: a written statement of any cooling-off or recusal rule for staff formerly employed by evaluated developers, and the start and end dates of the executive director's Anthropic employment as published on the about page." ], "graph_id": "assessment#741", "evidence_tier": "tier 2 (ledger)", "derived": { "leads": { "v": 2, "b": [ "palisade.10" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "palisade.10" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "palisade.10" ], "h": false, "c": false, "u": false, "x": [ "palisade.07" ] }, "spans": { "v": 2, "b": [ "palisade.10" ], "h": false, "c": false, "u": false, "x": [ "palisade.07" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "palisade.07", "palisade.10" ] } }, "rationale_derived": "Capped at 2 by palisade.10 (P.3): Founder and executive director previously worked on Anthropic's security team; SFF recommendations of about $3.2M.. Floors up to 2 from palisade.07 do not exceed the cap.", "rationale_leads": "Capped at 2 by palisade.10 (P.3): Founder and executive director previously worked on Anthropic's security team; SFF recommendations of about $3.2M.. Floors up to 2 from palisade.07 do not exceed the cap." }, "A": { "evaluator": "palisade", "dimension": "A", "value": 1, "anchor": 1, "signals": [ "palisade.05" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "mechanism": "lab-controlled", "graph_id": "assessment#742", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 1, "b": [ "palisade.05" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "palisade.05" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "palisade.05" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "palisade.05" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "palisade.05" ] } }, "rationale_derived": "Capped at 1 by palisade.05 (A.1): Little pre-deployment access; most work is on released models..", "rationale_leads": "Capped at 1 by palisade.05 (A.1): Little pre-deployment access; most work is on released models.." }, "S": { "evaluator": "palisade", "dimension": "S", "value": 3, "anchor": 3, "signals": [ "palisade.02" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#743", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "palisade.02" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "palisade.02" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "palisade.02" ] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "palisade.02" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "palisade.02" ] } }, "rationale_derived": "Floored at 3 by palisade.02 (S.5): Sets its own questions and publishes without lab review.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by palisade.02 (S.5): Sets its own questions and publishes without lab review.. No admissible signal caps it." }, "R": { "evaluator": "palisade", "dimension": "R", "value": 4, "anchor": 4, "signals": [ "palisade.03" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "mechanism": "self-imposed", "graph_id": "assessment#744", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 4, "b": [ "palisade.03" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 4, "b": [ "palisade.03" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 4, "b": [ "palisade.03" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 4, "b": [ "palisade.03" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "palisade.03" ] } }, "rationale_derived": "Floored at 4 by palisade.03 (R.5): Publishes findings unwelcome to labs, including shutdown resistance and self-replication.. No admissible signal caps it.", "rationale_leads": "Floored at 4 by palisade.03 (R.5): Publishes findings unwelcome to labs, including shutdown resistance and self-replication.. No admissible signal caps it." }, "M": { "evaluator": "palisade", "dimension": "M", "value": 3, "anchor": 3, "signals": [ "palisade.04" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#745", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "palisade.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "palisade.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "palisade.04" ] }, "spans": { "v": 3, "b": [ "palisade.04" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "palisade.04" ] } }, "rationale_derived": "Floored at 3 by palisade.04 (M.1): Publishes code and transcripts.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by palisade.04 (M.1): Publishes code and transcripts.. No admissible signal caps it." }, "X": { "evaluator": "palisade", "dimension": "X", "value": 3, "anchor": 3, "signals": [ "palisade.08", "palisade.12" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#746", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 3, "b": [ "palisade.12" ], "h": true, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "palisade.12" ], "h": true, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "palisade.12" ], "h": false, "c": false, "u": false, "x": [ "palisade.08" ] }, "spans": { "v": 3, "b": [ "palisade.12" ], "h": false, "c": false, "u": false, "x": [ "palisade.08" ] }, "primary": { "v": 3, "b": [ "palisade.12" ], "h": false, "c": false, "u": false, "x": [ "palisade.08" ] } }, "rationale_derived": "Capped at 3 by palisade.12 (X.7): Palisade's FY2024 Form 990 reports $344,829 (10.7% of revenue) from program services; the counterparty is not disclosed.. Floors up to 3 from palisade.08 do not exceed the cap.", "rationale_leads": "Capped at 3 by palisade.12 (X.7): Palisade's FY2024 Form 990 reports $344,829 (10.7% of revenue) from program services; the counterparty is not disclosed.. Floors up to 3 from palisade.08 do not exceed the cap." } }, "signals": [ { "id": "palisade.01", "evaluator": "palisade", "dimension": "F", "direction": "for", "claim": "Philanthropically funded; no lab contracts found.", "sources": [ "palisade", "propublica-palisade-990", "palisade-donate" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "F.11", "quote": "Contributions $2,887,428 89.3%", "quote_source": "propublica-palisade-990", "graph_id": "signal#474", "source_graph_ids": [ "source#149", "source#153", "source#146" ] }, { "id": "palisade.02", "evaluator": "palisade", "dimension": "S", "direction": "for", "claim": "Sets its own questions and publishes without lab review.", "sources": [ "palisade" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "S.5", "graph_id": "signal#475", "source_graph_ids": [ "source#149" ] }, { "id": "palisade.03", "evaluator": "palisade", "dimension": "R", "direction": "for", "claim": "Publishes findings unwelcome to labs, including shutdown resistance and self-replication.", "sources": [ "palisade", "computerworld-palisade-shutdown", "techrepublic-palisade-shutdown" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 4 }, "rule": "R.5", "quote": "the first time AI models have been observed preventing themselves from being shut down despite explicit instructions", "quote_source": "palisade", "graph_id": "signal#476", "source_graph_ids": [ "source#149", "source#54", "source#181" ] }, { "id": "palisade.04", "evaluator": "palisade", "dimension": "M", "direction": "for", "claim": "Publishes code and transcripts.", "sources": [ "palisade", "palisade-shutdown", "palisade-selfrep" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "M.1", "quote": "View Published Paper Code Access Paper", "quote_source": "palisade-shutdown", "graph_id": "signal#477", "source_graph_ids": [ "source#149", "source#148", "source#147" ] }, { "id": "palisade.05", "evaluator": "palisade", "dimension": "A", "direction": "against", "claim": "Little pre-deployment access; most work is on released models.", "sources": [ "palisade" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 1 }, "rule": "A.1", "graph_id": "signal#478", "source_graph_ids": [ "source#149" ] }, { "id": "palisade.06", "evaluator": "palisade", "dimension": "G", "direction": "for", "claim": "Nonprofit.", "sources": [ "palisade", "palisade-about" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "G.1", "quote": "Palisade Research is a nonprofit based in Berkeley, California", "quote_source": "palisade-about", "graph_id": "signal#479", "source_graph_ids": [ "source#149", "source#145" ] }, { "id": "palisade.07", "evaluator": "palisade", "dimension": "P", "direction": "for", "claim": "No lab board roles found.", "sources": [ "palisade" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "P.6", "graph_id": "signal#480", "source_graph_ids": [ "source#149" ] }, { "id": "palisade.08", "evaluator": "palisade", "dimension": "X", "direction": "for", "claim": "No commercial products.", "sources": [ "palisade" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 4 }, "rule": "X.7", "graph_id": "signal#481", "source_graph_ids": [ "source#149" ] }, { "id": "palisade.09", "evaluator": "palisade", "dimension": "F", "direction": "against", "claim": "About $3.8M in cumulative Coefficient Giving grants per the index snapshot; Coefficient's principal is an Anthropic Series A investor and board observer by his own account. No bounded negative on lab money is on file.", "sources": [ "coefficient-index", "metr-money-figure" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.3", "as_of": "2026-09-11", "bound": { "cap": 3 }, "rule": "F.6", "graph_id": "signal#482", "source_graph_ids": [ "source#52", "source#117" ] }, { "id": "palisade.10", "evaluator": "palisade", "dimension": "P", "direction": "against", "claim": "Founder and executive director previously worked on Anthropic's security team; SFF recommendations of about $3.2M.", "sources": [ "evaluators-ledger", "palisade-about" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-11", "bound": { "cap": 2 }, "rule": "P.3", "quote": "In 2022, Jeffrey was helping to build out the security team at Anthropic", "quote_source": "palisade-about", "graph_id": "signal#483", "source_graph_ids": [ "source#81", "source#145" ] }, { "id": "palisade.11", "evaluator": "palisade", "dimension": "F", "direction": "against", "claim": "Coefficient Giving's database records two grants totaling $2,123,463 for general support; the imported index total of $3,803,463 is marked differs pending reconciliation.", "sources": [ "op-palisade", "evaluators-ledger" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2026-09-15", "quote": "$2,123,463 2025-05", "bound": { "cap": 3 }, "rule": "F.6", "quote_source": "evaluators-ledger", "graph_id": "signal#484", "source_graph_ids": [ "source#139", "source#81" ] }, { "id": "palisade.12", "evaluator": "palisade", "dimension": "X", "direction": "against", "claim": "Palisade's FY2024 Form 990 reports $344,829 (10.7% of revenue) from program services; the counterparty is not disclosed.", "sources": [ "propublica-palisade-990" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "X.7", "quote": "Program Services $344,829 10.7%", "quote_source": "propublica-palisade-990", "note": "Rule gap: no X rule covers undisclosed program-service revenue. By anchor text a bounded 'no commercial products' (4) is not established while service revenue exists, so the cap is 3 pending a document request naming the counterparty; X.1 would cap at 2 only if the services were sold into the evaluated ecosystem.", "graph_id": "signal#485", "source_graph_ids": [ "source#153" ] }, { "id": "palisade.13", "evaluator": "palisade", "dimension": "F", "direction": "against", "claim": "SFF recommended $1,133,000 to Palisade in 2025 (plus a $928,000 matching pledge); SFF's funder Jaan Tallinn led Anthropic's Series A.", "sources": [ "sff-2025", "anthropic-series-a" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "F.6", "quote": "Palisade Research Main: $477,000 Freedom: $441,000 Fairness: $0 Mean: $216,000 $1,133,000", "quote_source": "sff-2025", "note": "Grants from a funder whose principal is an investor in a frontier developer cap at 3 (F.6). Ledger row T66 records the transfer; anthropic-series-a: 'The Series A round was led by Jaan Tallinn'.", "graph_id": "signal#486", "source_graph_ids": [ "source#176", "source#28" ] } ], "scores": { "lab": 64, "regulator": 64, "public": 70, "equal": 66 }, "scores_by_policy": { "leads": { "lab": { "score": 64, "band": "conditional", "coverage": 8, "range": [ 64, 64 ] }, "regulator": { "score": 64, "band": "conditional", "coverage": 8, "range": [ 64, 64 ] }, "public": { "score": 70, "band": "conditional", "coverage": 8, "range": [ 70, 70 ] }, "equal": { "score": 66, "band": "conditional", "coverage": 8, "range": [ 66, 66 ] } }, "standard": { "lab": { "score": 64, "band": "conditional", "coverage": 8, "range": [ 64, 64 ] }, "regulator": { "score": 64, "band": "conditional", "coverage": 8, "range": [ 64, 64 ] }, "public": { "score": 70, "band": "conditional", "coverage": 8, "range": [ 70, 70 ] }, "equal": { "score": 66, "band": "conditional", "coverage": 8, "range": [ 66, 66 ] } }, "against_interest": { "lab": { "score": 62, "band": "conditional", "coverage": 5, "range": [ 41, 74 ] }, "regulator": { "score": 60, "band": "conditional", "coverage": 5, "range": [ 36, 76 ] }, "public": { "score": 73, "band": "conditional", "coverage": 5, "range": [ 51, 81 ] }, "equal": { "score": 65, "band": "conditional", "coverage": 5, "range": [ 41, 78 ] } }, "spans": { "lab": { "score": 73, "band": "clear", "coverage": 6, "range": [ 47, 83 ] }, "regulator": { "score": 73, "band": "clear", "coverage": 6, "range": [ 44, 84 ] }, "public": { "score": 72, "band": "clear", "coverage": 6, "range": [ 61, 76 ] }, "equal": { "score": 71, "band": "clear", "coverage": 6, "range": [ 53, 78 ] } }, "primary": { "lab": { "score": 75, "band": "clear", "coverage": 2, "range": [ 17, 94 ] }, "regulator": { "score": 75, "band": "clear", "coverage": 2, "range": [ 11, 96 ] }, "public": { "score": 75, "band": "clear", "coverage": 2, "range": [ 23, 93 ] }, "equal": { "score": 75, "band": "clear", "coverage": 2, "range": [ 19, 94 ] } } }, "band": "conditional", "coverage": 8, "floor": "A", "what_moves": { "lab": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 69, "band": "conditional", "base": 64 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 66, "band": "conditional", "base": 64 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 67, "band": "conditional", "base": 64 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 69, "band": "clear", "base": 64 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 68, "band": "conditional", "base": 64 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 66, "band": "conditional", "base": 64 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 65, "band": "conditional", "base": 64 } ], "regulator": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 68, "band": "conditional", "base": 64 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 66, "band": "conditional", "base": 64 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 66, "band": "conditional", "base": 64 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 69, "band": "clear", "base": 64 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 69, "band": "conditional", "base": 64 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 66, "band": "conditional", "base": 64 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 64, "band": "conditional", "base": 64 } ], "public": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 76, "band": "conditional", "base": 70 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 74, "band": "conditional", "base": 70 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 74, "band": "conditional", "base": 70 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 71, "band": "clear", "base": 70 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 73, "band": "conditional", "base": 70 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 71, "band": "conditional", "base": 70 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 71, "band": "conditional", "base": 70 } ], "equal": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 69, "band": "conditional", "base": 66 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 69, "band": "conditional", "base": 66 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 69, "band": "conditional", "base": 66 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 69, "band": "clear", "base": 66 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 69, "band": "conditional", "base": 66 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 69, "band": "conditional", "base": 66 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 69, "band": "conditional", "base": 66 } ] } }, { "id": "rand", "name": "RAND", "type": "nonprofit", "hq": "Santa Monica, US", "domains": [ "bio", "cyber", "assurance" ], "confidence": "med", "summary": "Think tank with CBRN and security expertise; AEF founding member.", "what_would_move_the_score": "More public model-level findings.", "role": "referee", "dissent": { "lower": "Publication should be 1. RAND's frontier-model work reaches the public mainly through METR's Canary summaries and government-restricted reports; no RAND-authored adverse finding about a named lab's model is on record, and the RAND-Irregular model-theft paper is generic. Findings that surface only as another organization's summaries are R.3's lab-summarized citations in all but name.", "higher": "Publication should be 3. RAND's research-integrity page commits to free and open publication of findings, disclosure of every funding source, and policies for intellectual independence; thousands of reports are free on rand.org. What is withheld is classified by government, not redacted by any lab, and R.4 tags that mechanism statutory rather than as lab control." }, "list_group": "referee", "graph_id": "evaluator#223", "values": { "F": 3, "G": 2, "P": 2, "A": 3, "S": 3, "R": 2, "M": 2, "X": 3 }, "values_by_policy": { "leads": { "F": 3, "G": 2, "P": 2, "A": 3, "S": 3, "R": 2, "M": 2, "X": 3 }, "standard": { "F": 3, "G": 2, "P": 2, "A": 3, "S": 3, "R": 2, "M": 2, "X": 3 }, "against_interest": { "F": 3, "G": 2, "P": null, "A": null, "S": null, "R": 2, "M": 2, "X": null }, "spans": { "F": 3, "G": 2, "P": 2, "A": null, "S": null, "R": null, "M": null, "X": null }, "primary": { "F": 3, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "rand", "dimension": "F", "value": 3, "anchor": 3, "signals": [ "rand.03", "rand.09", "rand.10", "rand.11" ], "assessed": "2026-09-15", "assessor": "yohei/claude v1.0", "graph_id": "assessment#747", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 3, "b": [ "rand.03", "rand.11" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "rand.03", "rand.11" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "rand.03", "rand.11" ], "h": false, "c": false, "u": false, "x": [ "rand.10" ] }, "spans": { "v": 3, "b": [ "rand.03", "rand.11" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": 3, "b": [ "rand.09" ], "h": false, "c": false, "u": false, "x": [ "rand.03", "rand.10", "rand.11" ] } }, "rationale_derived": "Capped at 3 by rand.03 (F.6): Large share of AI work funded by Coefficient Giving.; capped at 3 by rand.11 (F.6): SFF recommended $1,022,000 to RAND's Technology and Security Policy Center in 2025; SFF's funder Jaan Tallinn led Anthropic's Series A.. Floors up to 3 from rand.09, rand.10 do not exceed the cap.", "rationale_leads": "Capped at 3 by rand.03 (F.6): Large share of AI work funded by Coefficient Giving.; capped at 3 by rand.11 (F.6): SFF recommended $1,022,000 to RAND's Technology and Security Policy Center in 2025; SFF's funder Jaan Tallinn led Anthropic's Series A.. Floors up to 3 from rand.09, rand.10 do not exceed the cap." }, "G": { "evaluator": "rand", "dimension": "G", "value": 2, "anchor": 2, "signals": [ "rand.01" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#748", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "rand.01" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "rand.01" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "rand.01" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "rand.01" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "rand.01" ] } }, "rationale_derived": "Floored at 2 by rand.01 (G.1): Government-contracted with long-standing COI and publication norms; AEF founding member.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by rand.01 (G.1): Government-contracted with long-standing COI and publication norms; AEF founding member.. No admissible signal caps it." }, "P": { "evaluator": "rand", "dimension": "P", "value": 2, "anchor": 2, "signals": [ "rand.05" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#749", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "rand.05" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "rand.05" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "rand.05" ] }, "spans": { "v": 2, "b": [ "rand.05" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "rand.05" ] } }, "rationale_derived": "Floored at 2 by rand.05 (P.6): Institutional COI rules.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by rand.05 (P.6): Institutional COI rules.. No admissible signal caps it." }, "A": { "evaluator": "rand", "dimension": "A", "value": 3, "anchor": 3, "signals": [ "rand.02" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "mechanism": "statutory", "graph_id": "assessment#750", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "rand.02" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "rand.02" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "rand.02" ] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "rand.02" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "rand.02" ] } }, "rationale_derived": "Floored at 3 by rand.02 (A.4): Access through government channels, including classified work.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by rand.02 (A.4): Access through government channels, including classified work.. No admissible signal caps it." }, "S": { "evaluator": "rand", "dimension": "S", "value": 3, "anchor": 3, "signals": [ "rand.06" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#751", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "rand.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "rand.06" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "rand.06" ] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "rand.06" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "rand.06" ] } }, "rationale_derived": "Floored at 3 by rand.06 (S.3): Sets own research agenda.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by rand.06 (S.3): Sets own research agenda.. No admissible signal caps it." }, "R": { "evaluator": "rand", "dimension": "R", "value": 2, "anchor": 2, "signals": [ "rand.04" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "mechanism": "statutory", "graph_id": "assessment#752", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "rand.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "rand.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "rand.04" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "rand.04" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "rand.04" ] } }, "rationale_derived": "Capped at 2 by rand.04 (R.4): Much output is government-restricted rather than public..", "rationale_leads": "Capped at 2 by rand.04 (R.4): Much output is government-restricted rather than public.." }, "M": { "evaluator": "rand", "dimension": "M", "value": 2, "anchor": 2, "signals": [ "rand.07" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#753", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "rand.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "rand.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "rand.07" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "rand.07" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "rand.07" ] } }, "rationale_derived": "Capped at 2 by rand.07 (M.2): Methods partly public..", "rationale_leads": "Capped at 2 by rand.07 (M.2): Methods partly public.." }, "X": { "evaluator": "rand", "dimension": "X", "value": 3, "anchor": 3, "signals": [ "rand.08" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "evidence_limited": true, "graph_id": "assessment#754", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "rand.08" ], "h": true, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "rand.08" ], "h": true, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "rand.08" ] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "rand.08" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "rand.08" ] } }, "rationale_derived": "Floored at 3 by rand.08 (X.7): No commercial products.. No admissible signal caps it. Held: C15: a 4 needs a quoted span; C16: a 4 needs a tier-1/2 source or two independent sources, one not self-published.", "rationale_leads": "Floored at 3 by rand.08 (X.7): No commercial products.. No admissible signal caps it. Held: C15: a 4 needs a quoted span; C16: a 4 needs a tier-1/2 source or two independent sources, one not self-published." } }, "signals": [ { "id": "rand.01", "evaluator": "rand", "dimension": "G", "direction": "for", "claim": "Government-contracted with long-standing COI and publication norms; AEF founding member.", "sources": [ "rand-aef", "aef-launch", "rand-integrity" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "G.1", "quote": "a policy of mandatory disclosure", "quote_source": "rand-integrity", "graph_id": "signal#487", "source_graph_ids": [ "source#157", "source#10", "source#159" ] }, { "id": "rand.02", "evaluator": "rand", "dimension": "A", "direction": "for", "claim": "Access through government channels, including classified work.", "sources": [ "rand-aef" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "A.4", "graph_id": "signal#488", "source_graph_ids": [ "source#157" ] }, { "id": "rand.03", "evaluator": "rand", "dimension": "F", "direction": "against", "claim": "Large share of AI work funded by Coefficient Giving.", "sources": [ "rand-aef", "coefficient-wiki", "anthropic-series-a", "evaluators-ledger" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 3 }, "rule": "F.6", "quote": "Board of directors Dustin Moskovitz, Cari Tuna, Divesh Makan, Holden Karnofsky, and Alexander Berger", "quote_source": "coefficient-wiki", "graph_id": "signal#489", "source_graph_ids": [ "source#157", "source#53", "source#28", "source#81" ] }, { "id": "rand.04", "evaluator": "rand", "dimension": "R", "direction": "against", "claim": "Much output is government-restricted rather than public.", "sources": [ "rand-aef" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "R.4", "graph_id": "signal#490", "source_graph_ids": [ "source#157" ] }, { "id": "rand.05", "evaluator": "rand", "dimension": "P", "direction": "for", "claim": "Institutional COI rules.", "sources": [ "rand-aef", "rand-integrity" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "P.6", "quote": "a policy of mandatory disclosure", "quote_source": "rand-integrity", "graph_id": "signal#491", "source_graph_ids": [ "source#157", "source#159" ] }, { "id": "rand.06", "evaluator": "rand", "dimension": "S", "direction": "for", "claim": "Sets own research agenda.", "sources": [ "rand-aef" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "S.3", "graph_id": "signal#492", "source_graph_ids": [ "source#157" ] }, { "id": "rand.07", "evaluator": "rand", "dimension": "M", "direction": "against", "claim": "Methods partly public.", "sources": [ "rand-aef" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "M.2", "graph_id": "signal#493", "source_graph_ids": [ "source#157" ] }, { "id": "rand.08", "evaluator": "rand", "dimension": "X", "direction": "for", "claim": "No commercial products.", "sources": [ "rand-aef" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 4 }, "rule": "X.7", "graph_id": "signal#494", "source_graph_ids": [ "source#157" ] }, { "id": "rand.09", "evaluator": "rand", "dimension": "F", "direction": "for", "claim": "FY2024 revenue of about $489M is mostly US government contract research; Coefficient's $87M since 2020 is a small share.", "sources": [ "evaluators-ledger", "propublica-rand-990" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-11", "bound": { "floor": 3 }, "rule": "F.9", "quote": "Contributions $488,777,692 91.4%", "quote_source": "propublica-rand-990", "graph_id": "signal#495", "source_graph_ids": [ "source#81", "source#154" ] }, { "id": "rand.10", "evaluator": "rand", "dimension": "F", "direction": "for", "claim": "The Audacious commitment to Canary is about $38 million across RAND and METR; RAND's Coefficient grants include $10.5M for emerging technology initiatives.", "sources": [ "rand-audacious-2024", "coefficient-index" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2024-10-09", "quote": "Canary—a Collaboration with METR—to Receive Approximately $38 Million", "bound": { "floor": 3 }, "rule": "F.6", "quote_source": "rand-audacious-2024", "graph_id": "signal#496", "source_graph_ids": [ "source#158", "source#52" ] }, { "id": "rand.11", "evaluator": "rand", "dimension": "F", "direction": "against", "claim": "SFF recommended $1,022,000 to RAND's Technology and Security Policy Center in 2025; SFF's funder Jaan Tallinn led Anthropic's Series A.", "sources": [ "sff-2025", "anthropic-series-a" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "F.6", "quote": "RAND Corporation [Technology and Security Policy Center] Main: $274,000 Freedom: $749,000", "quote_source": "sff-2025", "note": "Grants from a funder whose principal is an investor in a frontier developer cap at 3 (F.6). Ledger row T70; anthropic-series-a: 'The Series A round was led by Jaan Tallinn'. Immaterial against $534M revenue but rule-required.", "graph_id": "signal#497", "source_graph_ids": [ "source#176", "source#28" ] } ], "scores": { "lab": 65, "regulator": 64, "public": 61, "equal": 63 }, "scores_by_policy": { "leads": { "lab": { "score": 65, "band": "clear", "coverage": 8, "range": [ 65, 65 ] }, "regulator": { "score": 64, "band": "clear", "coverage": 8, "range": [ 64, 64 ] }, "public": { "score": 61, "band": "clear", "coverage": 8, "range": [ 61, 61 ] }, "equal": { "score": 63, "band": "clear", "coverage": 8, "range": [ 63, 63 ] } }, "standard": { "lab": { "score": 65, "band": "clear", "coverage": 8, "range": [ 65, 65 ] }, "regulator": { "score": 64, "band": "clear", "coverage": 8, "range": [ 64, 64 ] }, "public": { "score": 61, "band": "clear", "coverage": 8, "range": [ 61, 61 ] }, "equal": { "score": 63, "band": "clear", "coverage": 8, "range": [ 63, 63 ] } }, "against_interest": { "lab": { "score": 59, "band": "clear", "coverage": 4, "range": [ 29, 80 ] }, "regulator": { "score": 57, "band": "clear", "coverage": 4, "range": [ 29, 79 ] }, "public": { "score": 60, "band": "clear", "coverage": 4, "range": [ 39, 74 ] }, "equal": { "score": 56, "band": "clear", "coverage": 4, "range": [ 28, 78 ] } }, "spans": { "lab": { "score": 63, "band": "clear", "coverage": 3, "range": [ 23, 87 ] }, "regulator": { "score": 61, "band": "clear", "coverage": 3, "range": [ 21, 86 ] }, "public": { "score": 61, "band": "clear", "coverage": 3, "range": [ 34, 79 ] }, "equal": { "score": 58, "band": "clear", "coverage": 3, "range": [ 22, 84 ] } }, "primary": { "lab": { "score": 75, "band": "clear", "coverage": 1, "range": [ 14, 96 ] }, "regulator": { "score": 75, "band": "clear", "coverage": 1, "range": [ 11, 96 ] }, "public": { "score": 75, "band": "clear", "coverage": 1, "range": [ 19, 94 ] }, "equal": { "score": 75, "band": "clear", "coverage": 1, "range": [ 9, 97 ] } } }, "band": "clear", "coverage": 8, "floor": "G", "what_moves": { "lab": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 69, "band": "clear", "base": 65 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 67, "band": "clear", "base": 65 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 67, "band": "clear", "base": 65 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 70, "band": "clear", "base": 65 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 69, "band": "clear", "base": 65 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 68, "band": "clear", "base": 65 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 67, "band": "clear", "base": 65 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 66, "band": "clear", "base": 65 } ], "regulator": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 68, "band": "clear", "base": 64 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 66, "band": "clear", "base": 64 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 66, "band": "clear", "base": 64 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 69, "band": "clear", "base": 64 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 69, "band": "clear", "base": 64 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 68, "band": "clear", "base": 64 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 66, "band": "clear", "base": 64 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 64, "band": "clear", "base": 64 } ], "public": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 68, "band": "clear", "base": 61 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 65, "band": "clear", "base": 61 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 65, "band": "clear", "base": 61 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 63, "band": "clear", "base": 61 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 64, "band": "clear", "base": 61 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 66, "band": "clear", "base": 61 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 63, "band": "clear", "base": 61 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 63, "band": "clear", "base": 61 } ], "equal": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 66, "band": "clear", "base": 63 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 66, "band": "clear", "base": 63 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 66, "band": "clear", "base": 63 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 66, "band": "clear", "base": 63 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 66, "band": "clear", "base": 63 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 66, "band": "clear", "base": 63 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 66, "band": "clear", "base": 63 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 66, "band": "clear", "base": 63 } ] } }, { "id": "redwood", "name": "Redwood Research", "type": "nonprofit", "hq": "Berkeley, US", "domains": [ "scheming", "incident", "assurance" ], "confidence": "med", "summary": "AI control research group; contributed a contractor to the METR incident investigation.", "what_would_move_the_score": "A diversified funder list and a stated policy on co-authoring with labs it evaluates.", "role": "referee", "dissent": { "lower": "Personnel could be 1. Redwood's chief scientist co-authored the alignment-faking paper with Anthropic's Alignment Science team, an evaluation of Claude run jointly with the developer, and then worked inside OpenAI as a METR contractor; no recusal or cooling-off policy is published. Anchor 1 describes leaders with working roles at labs and informal recusal, and the record shows the roles but not the recusal.", "higher": "Personnel could be 3. No board, advisory, or equity position at a lab appears in the record; the OpenAI incident work took no payment beyond API credits; the co-authored paper was research, not an evaluation engagement, and Anthropic named four independent reviewers. Anchor 3 asks for a recusal policy and disclosed ties; the ties are disclosed in the paper itself, and a published recusal rule would complete the anchor." }, "list_group": "referee", "graph_id": "evaluator#224", "values": { "F": 3, "G": 2, "P": 2, "A": 4, "S": 3, "R": 3, "M": 3, "X": 3 }, "values_by_policy": { "leads": { "F": 3, "G": 2, "P": null, "A": 4, "S": 3, "R": 3, "M": 3, "X": 3 }, "standard": { "F": 3, "G": 2, "P": null, "A": 4, "S": 3, "R": 3, "M": 3, "X": 3 }, "against_interest": { "F": 3, "G": 2, "P": null, "A": 4, "S": 3, "R": 3, "M": 3, "X": 3 }, "spans": { "F": 3, "G": 2, "P": null, "A": 4, "S": 3, "R": 3, "M": null, "X": 3 }, "primary": { "F": 3, "G": 2, "P": null, "A": null, "S": null, "R": null, "M": null, "X": 3 } }, "assessments": { "F": { "evaluator": "redwood", "dimension": "F", "value": 3, "anchor": 3, "signals": [ "redwood.01", "redwood.02", "redwood.10", "redwood.11" ], "assessed": "2026-09-15", "assessor": "yohei/claude v1.0", "graph_id": "assessment#755", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 3, "b": [ "redwood.02", "redwood.10", "redwood.11" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "redwood.02", "redwood.11" ], "h": false, "c": false, "u": false, "x": [ "redwood.10" ] }, "against_interest": { "v": 3, "b": [ "redwood.02", "redwood.11" ], "h": false, "c": false, "u": false, "x": [ "redwood.10" ] }, "spans": { "v": 3, "b": [ "redwood.02", "redwood.11" ], "h": false, "c": false, "u": false, "x": [ "redwood.10" ] }, "primary": { "v": 3, "b": [ "redwood.11" ], "h": false, "c": false, "u": false, "x": [ "redwood.01", "redwood.02", "redwood.10" ] } }, "rationale_derived": "Capped at 3 by redwood.02 (F.6): Funding concentration: revenue nearly disappeared in one year when a few large grants ended.; capped at 3 by redwood.11 (F.6): Largest single grant: $36,566,000 from Coefficient Giving in November 2025 for AI control and alignment-faking work, after $10M in 2021 and $10M in 2022.. Floors up to 3 from redwood.01 do not exceed the cap. Not counted under this policy: redwood.10 (no confirmed source (imported)).", "rationale_leads": "Capped at 3 by redwood.02 (F.6): Funding concentration: revenue nearly disappeared in one year when a few large grants ended.; capped at 3 by redwood.10 (F.6): Coefficient grants of about $63M across five grants dominate income; SFF about $2.4M and a $1.33M Tallinn gift; both funders sit one to two steps from an Anthropic investor.; capped at 3 by redwood.11 (F.6): Largest single grant: $36,566,000 from Coefficient Giving in November 2025 for AI control and alignment-faking work, after $10M in 2021 and $10M in 2022.. Floors up to 3 from redwood.01 do not exceed the cap." }, "G": { "evaluator": "redwood", "dimension": "G", "value": 2, "anchor": 2, "signals": [ "redwood.06" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#756", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 2, "b": [ "redwood.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "redwood.06" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "redwood.06" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "redwood.06" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": 2, "b": [ "redwood.06" ], "h": false, "c": false, "u": false, "x": [] } }, "rationale_derived": "Floored at 2 by redwood.06 (G.1): Nonprofit.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by redwood.06 (G.1): Nonprofit.. No admissible signal caps it." }, "P": { "evaluator": "redwood", "dimension": "P", "value": 2, "anchor": 2, "signals": [ "redwood.05" ], "rationale": "Anchor 2 on personnel given the cited signals.", "assessed": "2026-09-14", "assessor": "yohei/claude v0", "unevidenced": true, "graph_id": "assessment#757", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [] }, "standard": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "redwood.05" ] } }, "rationale_derived": "Unevidenced under this policy: no admissible signal sets a bound.", "rationale_leads": "Unevidenced under this policy: no admissible signal sets a bound." }, "A": { "evaluator": "redwood", "dimension": "A", "value": 4, "anchor": 4, "signals": [ "redwood.03" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "lab-controlled", "graph_id": "assessment#758", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 4, "b": [ "redwood.03" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 4, "b": [ "redwood.03" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 4, "b": [ "redwood.03" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 4, "b": [ "redwood.03" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "redwood.03" ] } }, "rationale_derived": "Floored at 4 by redwood.03 (A.5): Contracted a staff member to METR for the on-site OpenAI investigation.. No admissible signal caps it.", "rationale_leads": "Floored at 4 by redwood.03 (A.5): Contracted a staff member to METR for the on-site OpenAI investigation.. No admissible signal caps it." }, "S": { "evaluator": "redwood", "dimension": "S", "value": 3, "anchor": 3, "signals": [ "redwood.07", "redwood.14", "redwood.15" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#759", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "redwood.14", "redwood.15" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "redwood.14", "redwood.15" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "redwood.14", "redwood.15" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "redwood.14", "redwood.15" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "redwood.07", "redwood.14", "redwood.15" ] } }, "rationale_derived": "Capped at 3 by redwood.14 (S.6): Redwood co-authored the December 2024 alignment-faking paper with Anthropic's Alignment Science team, an evaluation of Claude 3 Opus carried out with the developer whose models Redwood also evaluates.; capped at 3 by redwood.15 (S.1): In the OpenAI incident investigation, on which a Redwood staff member worked as a METR contractor, OpenAI defined the investigation period and the compromise of its own infrastructure was out of scope.. Floors up to 3 from redwood.07 do not exceed the cap.", "rationale_leads": "Capped at 3 by redwood.14 (S.6): Redwood co-authored the December 2024 alignment-faking paper with Anthropic's Alignment Science team, an evaluation of Claude 3 Opus carried out with the developer whose models Redwood also evaluates.; capped at 3 by redwood.15 (S.1): In the OpenAI incident investigation, on which a Redwood staff member worked as a METR contractor, OpenAI defined the investigation period and the compromise of its own infrastructure was out of scope.. Floors up to 3 from redwood.07 do not exceed the cap." }, "R": { "evaluator": "redwood", "dimension": "R", "value": 3, "anchor": 3, "signals": [ "redwood.04", "redwood.16" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "mechanism": "lab-controlled", "graph_id": "assessment#760", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "redwood.16" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "redwood.16" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "redwood.16" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "redwood.16" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "redwood.04", "redwood.16" ] } }, "rationale_derived": "Capped at 3 by redwood.16 (R.2): On the co-authored OpenAI incident report, OpenAI could redact any non-public information and gave feedback on structure, emphasis, clarity and tone that the authors incorporated and disclosed.. Floors up to 2 from redwood.04 do not exceed the cap.", "rationale_leads": "Capped at 3 by redwood.16 (R.2): On the co-authored OpenAI incident report, OpenAI could redact any non-public information and gave feedback on structure, emphasis, clarity and tone that the authors incorporated and disclosed.. Floors up to 2 from redwood.04 do not exceed the cap." }, "M": { "evaluator": "redwood", "dimension": "M", "value": 3, "anchor": 3, "signals": [ "redwood.08" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#761", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 3, "b": [ "redwood.08" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "redwood.08" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "redwood.08" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "redwood.08" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "redwood.08" ] } }, "rationale_derived": "Floored at 3 by redwood.08 (M.1): Publishes control methods and code.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by redwood.08 (M.1): Publishes control methods and code.. No admissible signal caps it." }, "X": { "evaluator": "redwood", "dimension": "X", "value": 3, "anchor": 3, "signals": [ "redwood.09", "redwood.12", "redwood.13" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#762", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 3, "b": [ "redwood.13" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "redwood.13" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "redwood.13" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "redwood.13" ], "h": false, "c": false, "u": false, "x": [ "redwood.09" ] }, "primary": { "v": 3, "b": [ "redwood.13" ], "h": false, "c": false, "u": false, "x": [ "redwood.09", "redwood.12" ] } }, "rationale_derived": "Capped at 3 by redwood.13 (X.8): ProPublica's extract of the Form 990 for Redwood Research Group Inc (EIN 87-1702255) shows program service revenue of $0 in FY2024; FY2023 shows $343,734 of program service revenue (3.4% of $10.0M) with the payer not identified in the summary.. Floors up to 3 from redwood.09, redwood.12 do not exceed the cap.", "rationale_leads": "Capped at 3 by redwood.13 (X.8): ProPublica's extract of the Form 990 for Redwood Research Group Inc (EIN 87-1702255) shows program service revenue of $0 in FY2024; FY2023 shows $343,734 of program service revenue (3.4% of $10.0M) with the payer not identified in the summary.. Floors up to 3 from redwood.09, redwood.12 do not exceed the cap." } }, "signals": [ { "id": "redwood.01", "evaluator": "redwood", "dimension": "F", "direction": "for", "claim": "No lab revenue; historically about $25M from Open Philanthropy.", "sources": [ "itbb-watchdogs" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "F.12", "quote": "Open Philanthropy's grants database shows approximately $25 million in historical funding to Redwood Research", "quote_source": "itbb-watchdogs", "graph_id": "signal#498", "source_graph_ids": [ "source#101" ] }, { "id": "redwood.02", "evaluator": "redwood", "dimension": "F", "direction": "against", "claim": "Funding concentration: revenue nearly disappeared in one year when a few large grants ended.", "sources": [ "itbb-watchdogs" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 3 }, "rule": "F.6", "quote": "a few large grants can make one year's revenue almost disappear in the next", "quote_source": "itbb-watchdogs", "graph_id": "signal#499", "source_graph_ids": [ "source#101" ] }, { "id": "redwood.03", "evaluator": "redwood", "dimension": "A", "direction": "for", "claim": "Contracted a staff member to METR for the on-site OpenAI investigation.", "sources": [ "metr-hf-investigation", "techtimes-metr-hf" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 4 }, "rule": "A.5", "quote": "a Redwood Research staff member contracting with METR", "quote_source": "metr-hf-investigation", "graph_id": "signal#500", "source_graph_ids": [ "source#116", "source#182" ] }, { "id": "redwood.04", "evaluator": "redwood", "dimension": "R", "direction": "for", "claim": "Co-authored the incident report with a redaction statement.", "sources": [ "metr-hf-investigation" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "R.7", "quote": "OpenAI agreed at the outset with METR and Redwood that we would be able to describe high-level scope and terms", "quote_source": "metr-hf-investigation", "graph_id": "signal#501", "source_graph_ids": [ "source#116" ] }, { "id": "redwood.05", "evaluator": "redwood", "dimension": "P", "direction": "against", "claim": "Co-authored alignment research with Anthropic; close talent flow with lab safety teams.", "sources": [ "itbb-watchdogs", "anthropic-alignment-faking" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": null, "rule": "P.3", "bound_note": "Section 9 (mis-dimensioned facts): the supported fact, co-authoring with Anthropic, belongs to S and is re-recorded as redwood.14; the P fact (talent flow with lab safety teams) has no source, so the signal is informational on P.", "quote": "in collaboration with Redwood Research , provides the first empirical example of a large language model", "quote_source": "anthropic-alignment-faking", "graph_id": "signal#502", "source_graph_ids": [ "source#101", "source#25" ] }, { "id": "redwood.06", "evaluator": "redwood", "dimension": "G", "direction": "for", "claim": "Nonprofit.", "sources": [ "itbb-watchdogs", "propublica-redwood" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "G.1", "quote": "Designated as a 501(c)(3)", "quote_source": "propublica-redwood", "graph_id": "signal#503", "source_graph_ids": [ "source#101", "source#155" ] }, { "id": "redwood.07", "evaluator": "redwood", "dimension": "S", "direction": "for", "claim": "Sets its own research agenda; incident work under METR terms.", "sources": [ "metr-hf-investigation" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "S.5", "quote": "a Redwood Research staff member contracting with METR", "quote_source": "metr-hf-investigation", "graph_id": "signal#504", "source_graph_ids": [ "source#116" ] }, { "id": "redwood.08", "evaluator": "redwood", "dimension": "M", "direction": "for", "claim": "Publishes control methods and code.", "sources": [ "itbb-watchdogs" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "M.1", "graph_id": "signal#505", "source_graph_ids": [ "source#101" ] }, { "id": "redwood.09", "evaluator": "redwood", "dimension": "X", "direction": "for", "claim": "No commercial products.", "sources": [ "itbb-watchdogs" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "X.7", "graph_id": "signal#506", "source_graph_ids": [ "source#101" ] }, { "id": "redwood.10", "evaluator": "redwood", "dimension": "F", "direction": "against", "claim": "Coefficient grants of about $63M across five grants dominate income; SFF about $2.4M and a $1.33M Tallinn gift; both funders sit one to two steps from an Anthropic investor.", "sources": [ "evaluators-ledger", "coefficient-index" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-11", "bound": { "cap": 3 }, "rule": "F.6", "graph_id": "signal#507", "source_graph_ids": [ "source#81", "source#52" ] }, { "id": "redwood.11", "evaluator": "redwood", "dimension": "F", "direction": "against", "claim": "Largest single grant: $36,566,000 from Coefficient Giving in November 2025 for AI control and alignment-faking work, after $10M in 2021 and $10M in 2022.", "sources": [ "op-redwood-general-support", "tnw-coefficient-ipo" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2025-11", "bound": { "cap": 3 }, "rule": "F.6", "quote": "listed in grant records at $36,566,000 for work on AI control and alignment faking", "quote_source": "tnw-coefficient-ipo", "graph_id": "signal#508", "source_graph_ids": [ "source#140", "source#186" ] }, { "id": "redwood.12", "evaluator": "redwood", "dimension": "X", "direction": "for", "claim": "The OpenAI incident investigation report states no payment was accepted other than API credits used in the investigation.", "sources": [ "zvi-hf-postmortem" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2026-08", "quote": "No payment was accepted, other than API credits used in the investigation", "bound": { "floor": 3 }, "rule": "X.4", "quote_source": "zvi-hf-postmortem", "graph_id": "signal#509", "source_graph_ids": [ "source#201" ] }, { "id": "redwood.13", "evaluator": "redwood", "dimension": "X", "direction": "against", "claim": "ProPublica's extract of the Form 990 for Redwood Research Group Inc (EIN 87-1702255) shows program service revenue of $0 in FY2024; FY2023 shows $343,734 of program service revenue (3.4% of $10.0M) with the payer not identified in the summary.", "sources": [ "propublica-redwood" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "X.8", "quote": "Contributions $9,659 43.8% Program Services $0", "quote_source": "propublica-redwood", "note": "X.8: program-service revenue in a filing whose clients are not disclosed (FY2023, $343,734) caps at 3 until the clients are named; if a lab is among them X.2 applies (1). FY2024 shows $0. Document request: the FY2023 program-service revenue by payer.", "graph_id": "signal#510", "source_graph_ids": [ "source#155" ] }, { "id": "redwood.14", "evaluator": "redwood", "dimension": "S", "direction": "against", "claim": "Redwood co-authored the December 2024 alignment-faking paper with Anthropic's Alignment Science team, an evaluation of Claude 3 Opus carried out with the developer whose models Redwood also evaluates.", "sources": [ "redwood-blog-af", "anthropic-alignment-faking" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "S.6", "quote": "We have a new paper (done in collaboration with Anthropic)", "quote_source": "redwood-blog-af", "note": "S.6: co-authoring research with a lab while evaluating it caps at 3. Both sources are self-published (one by each party) and confirmed.", "graph_id": "signal#511", "source_graph_ids": [ "source#160", "source#25" ] }, { "id": "redwood.16", "evaluator": "redwood", "dimension": "R", "direction": "against", "claim": "On the co-authored OpenAI incident report, OpenAI could redact any non-public information and gave feedback on structure, emphasis, clarity and tone that the authors incorporated and disclosed.", "sources": [ "metr-hf-investigation" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "R.2", "quote": "we made corrections and edits to structure, emphasis, clarity, and tone based on that feedback", "quote_source": "metr-hf-investigation", "note": "R.2: disclosed editorial feedback caps at 3. Recorded on Redwood because the same fact caps METR (metr.08) on the same report.", "graph_id": "signal#512", "source_graph_ids": [ "source#116" ] }, { "id": "redwood.15", "evaluator": "redwood", "dimension": "S", "direction": "against", "claim": "In the OpenAI incident investigation, on which a Redwood staff member worked as a METR contractor, OpenAI defined the investigation period and the compromise of its own infrastructure was out of scope.", "sources": [ "metr-hf-investigation" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "S.1", "quote": "OpenAI defined the investigation period as June 26th through July 13th", "quote_source": "metr-hf-investigation", "note": "S.1: a lab-set window in the most recent major engagement caps at 3. Recorded on Redwood because the same fact already caps METR (metr.06).", "graph_id": "signal#513", "source_graph_ids": [ "source#116" ] } ], "scores": { "lab": 78, "regulator": 78, "public": 72, "equal": 75 }, "scores_by_policy": { "leads": { "lab": { "score": 78, "band": "clear", "coverage": 7, "range": [ 71, 81 ] }, "regulator": { "score": 78, "band": "clear", "coverage": 7, "range": [ 70, 80 ] }, "public": { "score": 72, "band": "clear", "coverage": 7, "range": [ 61, 76 ] }, "equal": { "score": 75, "band": "clear", "coverage": 7, "range": [ 66, 78 ] } }, "standard": { "lab": { "score": 78, "band": "clear", "coverage": 7, "range": [ 71, 81 ] }, "regulator": { "score": 78, "band": "clear", "coverage": 7, "range": [ 70, 80 ] }, "public": { "score": 72, "band": "clear", "coverage": 7, "range": [ 61, 76 ] }, "equal": { "score": 75, "band": "clear", "coverage": 7, "range": [ 66, 78 ] } }, "against_interest": { "lab": { "score": 78, "band": "clear", "coverage": 7, "range": [ 71, 81 ] }, "regulator": { "score": 78, "band": "clear", "coverage": 7, "range": [ 70, 80 ] }, "public": { "score": 72, "band": "clear", "coverage": 7, "range": [ 61, 76 ] }, "equal": { "score": 75, "band": "clear", "coverage": 7, "range": [ 66, 78 ] } }, "spans": { "lab": { "score": 79, "band": "clear", "coverage": 6, "range": [ 64, 83 ] }, "regulator": { "score": 78, "band": "clear", "coverage": 6, "range": [ 63, 83 ] }, "public": { "score": 72, "band": "clear", "coverage": 6, "range": [ 57, 78 ] }, "equal": { "score": 75, "band": "clear", "coverage": 6, "range": [ 56, 81 ] } }, "primary": { "lab": { "score": 69, "band": "clear", "coverage": 3, "range": [ 21, 90 ] }, "regulator": { "score": 65, "band": "clear", "coverage": 3, "range": [ 16, 91 ] }, "public": { "score": 67, "band": "clear", "coverage": 3, "range": [ 30, 85 ] }, "equal": { "score": 67, "band": "clear", "coverage": 3, "range": [ 25, 88 ] } } }, "band": "clear", "coverage": 7, "floor": "G", "what_moves": { "lab": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 83, "band": "clear", "base": 78 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 81, "band": "clear", "base": 78 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 83, "band": "clear", "base": 78 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 82, "band": "clear", "base": 78 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 81, "band": "clear", "base": 78 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 80, "band": "clear", "base": 78 } ], "regulator": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 82, "band": "clear", "base": 78 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 81, "band": "clear", "base": 78 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 83, "band": "clear", "base": 78 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 82, "band": "clear", "base": 78 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 81, "band": "clear", "base": 78 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 78 } ], "public": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 79, "band": "clear", "base": 72 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 76, "band": "clear", "base": 72 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 72 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 72 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 74, "band": "clear", "base": 72 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 74, "band": "clear", "base": 72 } ], "equal": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 79, "band": "clear", "base": 75 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 79, "band": "clear", "base": 75 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 79, "band": "clear", "base": 75 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 79, "band": "clear", "base": 75 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 79, "band": "clear", "base": 75 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 79, "band": "clear", "base": 75 } ] } }, { "id": "saferai", "name": "SaferAI", "type": "nonprofit", "hq": "Paris, FR", "domains": [ "assurance", "cyber", "bio", "misuse" ], "confidence": "med", "summary": "Rates lab risk-management frameworks in public; runs EU Code of Practice evaluations.", "what_would_move_the_score": "Documented pre-release access under Code terms.", "role": "referee", "dissent": { "lower": "Access could be 0. No source records any pre-release model access for SaferAI: the goal-directedness evaluation it co-authored with Google DeepMind researchers tested models from Google DeepMind, OpenAI and Anthropic through ordinary means, and its own site describes framework ratings, risk models and advisory work rather than model testing. Anchor 0 is public API only, and that is the whole documented record.", "higher": "Access could be 2. SaferAI leads the risk-modelling workstream of the EU AI Office's CBRN lot under a three-year contract, participated in all four Code of Practice working groups, and co-authors with a frontier developer's researchers; the Code contemplates external evaluator access to models before release. Documented pre-release access under Code terms, which the card names as what would move the score, would reach anchor 1 or 2." }, "list_group": "referee", "graph_id": "evaluator#225", "values": { "F": 3, "G": 2, "P": 2, "A": 1, "S": 3, "R": 2, "M": 2, "X": 2 }, "values_by_policy": { "leads": { "F": 3, "G": 2, "P": 2, "A": 1, "S": 3, "R": 2, "M": 2, "X": 2 }, "standard": { "F": 3, "G": 2, "P": 2, "A": 1, "S": 3, "R": 2, "M": 2, "X": 2 }, "against_interest": { "F": 3, "G": null, "P": null, "A": 1, "S": 3, "R": 2, "M": null, "X": 2 }, "spans": { "F": 3, "G": 2, "P": null, "A": null, "S": 3, "R": 2, "M": null, "X": 2 }, "primary": { "F": null, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "saferai", "dimension": "F", "value": 3, "anchor": 3, "signals": [ "saferai.03", "saferai.12" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#763", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "saferai.12" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "saferai.12" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "saferai.12" ], "h": false, "c": false, "u": false, "x": [ "saferai.03" ] }, "spans": { "v": 3, "b": [ "saferai.12" ], "h": false, "c": false, "u": false, "x": [ "saferai.03" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "saferai.03", "saferai.12" ] } }, "rationale_derived": "Capped at 3 by saferai.12 (F.6): SFF recommended $311,000 to SaferAI in 2025 (with a $111,000 match) after earlier rounds; SFF's funder is Jaan Tallinn, the Anthropic Series A lead.. Floors up to 2 from saferai.03 do not exceed the cap.", "rationale_leads": "Capped at 3 by saferai.12 (F.6): SFF recommended $311,000 to SaferAI in 2025 (with a $111,000 match) after earlier rounds; SFF's funder is Jaan Tallinn, the Anthropic Series A lead.. Floors up to 2 from saferai.03 do not exceed the cap." }, "G": { "evaluator": "saferai", "dimension": "G", "value": 2, "anchor": 2, "signals": [ "saferai.05" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#764", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "saferai.05" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "saferai.05" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "saferai.05" ] }, "spans": { "v": 2, "b": [ "saferai.05" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "saferai.05" ] } }, "rationale_derived": "Floored at 2 by saferai.05 (G.1): Nonprofit.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by saferai.05 (G.1): Nonprofit.. No admissible signal caps it." }, "P": { "evaluator": "saferai", "dimension": "P", "value": 2, "anchor": 2, "signals": [ "saferai.06" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#765", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "saferai.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "saferai.06" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "saferai.06" ] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "saferai.06" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "saferai.06" ] } }, "rationale_derived": "Floored at 2 by saferai.06 (P.6): No lab roles found.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by saferai.06 (P.6): No lab roles found.. No admissible signal caps it." }, "A": { "evaluator": "saferai", "dimension": "A", "value": 1, "anchor": 1, "signals": [ "saferai.04" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "unknown", "graph_id": "assessment#766", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 1, "b": [ "saferai.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "saferai.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "saferai.04" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "saferai.04" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "saferai.04" ] } }, "rationale_derived": "Capped at 1 by saferai.04 (A.1): Limited pre-deployment model access to date..", "rationale_leads": "Capped at 1 by saferai.04 (A.1): Limited pre-deployment model access to date.." }, "S": { "evaluator": "saferai", "dimension": "S", "value": 3, "anchor": 3, "signals": [ "saferai.02", "saferai.13" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#767", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "saferai.13" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "saferai.13" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "saferai.13" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "saferai.13" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "saferai.02", "saferai.13" ] } }, "rationale_derived": "Capped at 3 by saferai.13 (S.6): SaferAI's executive director and founder co-authored \"Evaluating the Goal-Directedness of Large Language Models\" (April 2025) with Google DeepMind researchers, an evaluation of models from Google DeepMind, OpenAI and Anthropic, while SaferAI publicly rates Google DeepMind's risk-management framework.. Floors up to 3 from saferai.02 do not exceed the cap.", "rationale_leads": "Capped at 3 by saferai.13 (S.6): SaferAI's executive director and founder co-authored \"Evaluating the Goal-Directedness of Large Language Models\" (April 2025) with Google DeepMind researchers, an evaluation of models from Google DeepMind, OpenAI and Anthropic, while SaferAI publicly rates Google DeepMind's risk-management framework.. Floors up to 3 from saferai.02 do not exceed the cap." }, "R": { "evaluator": "saferai", "dimension": "R", "value": 2, "anchor": 2, "signals": [ "saferai.01" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "self-imposed", "graph_id": "assessment#768", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "saferai.01" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "saferai.01" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "saferai.01" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "saferai.01" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "saferai.01" ] } }, "rationale_derived": "Floored at 2 by saferai.01 (R.7): Publishes critical ratings of lab frameworks with scores attached.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by saferai.01 (R.7): Publishes critical ratings of lab frameworks with scores attached.. No admissible signal caps it." }, "M": { "evaluator": "saferai", "dimension": "M", "value": 2, "anchor": 2, "signals": [ "saferai.07" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#769", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "saferai.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "saferai.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "saferai.07" ] }, "spans": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "saferai.07" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "saferai.07" ] } }, "rationale_derived": "Floored at 2 by saferai.07 (M.2): Rating methodology published.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by saferai.07 (M.2): Rating methodology published.. No admissible signal caps it." }, "X": { "evaluator": "saferai", "dimension": "X", "value": 2, "anchor": 2, "signals": [ "saferai.08", "saferai.09", "saferai.11" ], "rationale": "Anchor 2: paid advisory work for at least one major AI company while rating companies' frameworks in public. If the unnamed client is not a frontier developer, restore to 4.", "assessed": "2026-09-15", "assessor": "yohei/claude v1.0", "open_questions": [ "Identity of the major AI company client.", "A published lab-money policy." ], "resolution": { "rule": "X.3", "value": 2, "note": "SaferAI's own page records framework work for G42 and for a major AI company; consulting for a lab caps at 2 even with no products." }, "graph_id": "assessment#770", "evidence_tier": "tier 2 (ledger)", "derived": { "leads": { "v": 2, "b": [ "saferai.09", "saferai.11" ], "h": false, "c": true, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "saferai.11" ], "h": false, "c": true, "u": false, "x": [ "saferai.09" ] }, "against_interest": { "v": 2, "b": [ "saferai.11" ], "h": false, "c": false, "u": false, "x": [ "saferai.08", "saferai.09" ] }, "spans": { "v": 2, "b": [ "saferai.11" ], "h": false, "c": false, "u": false, "x": [ "saferai.08", "saferai.09" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "saferai.08", "saferai.09", "saferai.11" ] } }, "rationale_derived": "Conflict: floor 3 from saferai.08 (X.4) against cap 2 from saferai.11 (X.3); resolved at 2 by X.3: SaferAI's own page records framework work for G42 and for a major AI company; consulting for a lab caps at 2 even with no products. Not counted under this policy: saferai.09 (no confirmed source (imported)).", "rationale_leads": "Conflict: floor 3 from saferai.08 (X.4) against cap 2 from saferai.09, saferai.11 (X.3, X.3); resolved at 2 by X.3: SaferAI's own page records framework work for G42 and for a major AI company; consulting for a lab caps at 2 even with no products." } }, "signals": [ { "claim": "Publishes critical ratings of lab frameworks with scores attached.", "curator": "yohei/claude v0", "dimension": "R", "direction": "for", "evaluator": "saferai", "id": "saferai.01", "recorded": "2026-09-14", "sources": [ "saferai", "time-saferai-ratings" ], "bound": { "floor": 2 }, "rule": "R.7", "quote": "We independently evaluate and rate how well leading AI companies manage risks", "quote_source": "saferai", "graph_id": "signal#514", "source_graph_ids": [ "source#163", "source#185" ] }, { "claim": "Part of the EU AI Office evaluation consortium; sets its own rating methodology.", "curator": "yohei/claude v0", "dimension": "S", "direction": "for", "evaluator": "saferai", "id": "saferai.02", "recorded": "2026-09-14", "sources": [ "longtermwiki-far", "saferai" ], "bound": { "floor": 3 }, "rule": "S.5", "quote": "We actively shaped the EU's Code of Practice for general-purpose AI models", "quote_source": "saferai", "graph_id": "signal#515", "source_graph_ids": [ "source#106", "source#163" ] }, { "claim": "No lab revenue found.", "curator": "yohei/claude v0", "dimension": "F", "direction": "for", "evaluator": "saferai", "id": "saferai.03", "recorded": "2026-09-14", "sources": [ "saferai" ], "bound": { "floor": 2 }, "rule": "F.10", "graph_id": "signal#516", "source_graph_ids": [ "source#163" ] }, { "claim": "Limited pre-deployment model access to date.", "curator": "yohei/claude v0", "dimension": "A", "direction": "against", "evaluator": "saferai", "id": "saferai.04", "recorded": "2026-09-14", "sources": [ "saferai" ], "bound": { "cap": 1 }, "rule": "A.1", "graph_id": "signal#517", "source_graph_ids": [ "source#163" ] }, { "claim": "Nonprofit.", "curator": "yohei/claude v0", "dimension": "G", "direction": "for", "evaluator": "saferai", "id": "saferai.05", "recorded": "2026-09-14", "sources": [ "saferai" ], "bound": { "floor": 2 }, "rule": "G.1", "quote": "SaferAI, a registered non-profit (reg no. 919 185 199 00016", "quote_source": "saferai", "graph_id": "signal#518", "source_graph_ids": [ "source#163" ] }, { "claim": "No lab roles found.", "curator": "yohei/claude v0", "dimension": "P", "direction": "for", "evaluator": "saferai", "id": "saferai.06", "recorded": "2026-09-14", "sources": [ "saferai" ], "bound": { "floor": 2 }, "rule": "P.6", "graph_id": "signal#519", "source_graph_ids": [ "source#163" ] }, { "claim": "Rating methodology published.", "curator": "yohei/claude v0", "dimension": "M", "direction": "for", "evaluator": "saferai", "id": "saferai.07", "recorded": "2026-09-14", "sources": [ "saferai" ], "bound": { "floor": 2 }, "rule": "M.2", "graph_id": "signal#520", "source_graph_ids": [ "source#163" ] }, { "claim": "No commercial products.", "curator": "yohei/claude v0", "dimension": "X", "direction": "for", "evaluator": "saferai", "id": "saferai.08", "recorded": "2026-09-14", "sources": [ "saferai" ], "bound": { "floor": 3 }, "rule": "X.4", "graph_id": "signal#521", "source_graph_ids": [ "source#163" ] }, { "as_of": "2026-09-11", "claim": "Advisory work for G42 and for an unnamed major AI company, alongside public framework ratings; no lab-money policy on the site.", "curator": "yohei/claude v0.4", "dimension": "X", "direction": "against", "evaluator": "saferai", "id": "saferai.09", "recorded": "2026-09-14", "sources": [ "evaluators-ledger" ], "bound": { "cap": 2 }, "rule": "X.3", "quote": "advised G42 and 'a major AI company' on frontier safety frameworks", "quote_source": "evaluators-ledger", "graph_id": "signal#522", "source_graph_ids": [ "source#81" ] }, { "as_of": "2026-09-11", "claim": "SFF recommendations of about $1.06M over 2022 to 2025; EU AI Office Lot 1 consortium member.", "curator": "yohei/claude v0.4", "dimension": "F", "direction": "for", "evaluator": "saferai", "id": "saferai.10", "recorded": "2026-09-14", "sources": [ "evaluators-ledger", "ted-864574", "sff-2025" ], "bound": { "floor": 3 }, "rule": "F.12", "quote": "SaferAI Main: $0 Freedom: $307,000 Fairness: $4,000 Mean: $0 $311,000", "quote_source": "sff-2025", "graph_id": "signal#523", "source_graph_ids": [ "source#81", "source#184", "source#176" ] }, { "as_of": "2026-09-15", "claim": "SaferAI says it contributed to frameworks for G42 and a major AI company and co-drafted G42's Frontier AI Safety policy; G42 names METR and SaferAI as supporting experts.", "curator": "yohei/claude v1.0", "dimension": "X", "direction": "against", "evaluator": "saferai", "id": "saferai.11", "quote": "We've contributed to frameworks for G42 and a major AI company", "recorded": "2026-09-15", "sources": [ "saferai-about" ], "bound": { "cap": 2 }, "rule": "X.3", "quote_source": "saferai-about", "graph_id": "signal#524", "source_graph_ids": [ "source#162" ] }, { "id": "saferai.12", "evaluator": "saferai", "dimension": "F", "direction": "against", "claim": "SFF recommended $311,000 to SaferAI in 2025 (with a $111,000 match) after earlier rounds; SFF's funder is Jaan Tallinn, the Anthropic Series A lead.", "sources": [ "sff-2025" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "F.6", "quote": "SaferAI Main: $0 Freedom: $307,000 Fairness: $4,000 Mean: $0 $311,000", "quote_source": "sff-2025", "note": "F.6: grants from a funder whose principal is an investor in a frontier developer cap at 3 (ledger R01, R08 confirmed).", "graph_id": "signal#525", "source_graph_ids": [ "source#176" ] }, { "id": "saferai.13", "evaluator": "saferai", "dimension": "S", "direction": "against", "claim": "SaferAI's executive director and founder co-authored \"Evaluating the Goal-Directedness of Large Language Models\" (April 2025) with Google DeepMind researchers, an evaluation of models from Google DeepMind, OpenAI and Anthropic, while SaferAI publicly rates Google DeepMind's risk-management framework.", "sources": [ "arxiv-goal-directedness", "saferai-about" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "S.6", "quote": "Authors: Tom Everitt , Cristina Garbacea , Alexis Bellot , Jonathan Richens , Henry Papadatos , Siméon Campos", "quote_source": "arxiv-goal-directedness", "note": "S.6: co-authoring research with a lab while evaluating it caps at 3. The same paper establishes a published evaluation of frontier models, so section 9 (no published evaluation of a frontier model) does not apply to SaferAI. Affiliations: the about page lists Henry Papadatos as Executive Director; the ledger lists Siméon Campos as founder and board chair.", "graph_id": "signal#526", "source_graph_ids": [ "source#38", "source#162" ] } ], "scores": { "lab": 54, "regulator": 54, "public": 57, "equal": 53 }, "scores_by_policy": { "leads": { "lab": { "score": 54, "band": "conditional", "coverage": 8, "range": [ 54, 54 ] }, "regulator": { "score": 54, "band": "conditional", "coverage": 8, "range": [ 54, 54 ] }, "public": { "score": 57, "band": "conditional", "coverage": 8, "range": [ 57, 57 ] }, "equal": { "score": 53, "band": "conditional", "coverage": 8, "range": [ 53, 53 ] } }, "standard": { "lab": { "score": 54, "band": "conditional", "coverage": 8, "range": [ 54, 54 ] }, "regulator": { "score": 54, "band": "conditional", "coverage": 8, "range": [ 54, 54 ] }, "public": { "score": 57, "band": "conditional", "coverage": 8, "range": [ 57, 57 ] }, "equal": { "score": 53, "band": "conditional", "coverage": 8, "range": [ 53, 53 ] } }, "against_interest": { "lab": { "score": 55, "band": "conditional", "coverage": 5, "range": [ 40, 67 ] }, "regulator": { "score": 55, "band": "conditional", "coverage": 5, "range": [ 39, 69 ] }, "public": { "score": 62, "band": "conditional", "coverage": 5, "range": [ 40, 75 ] }, "equal": { "score": 55, "band": "conditional", "coverage": 5, "range": [ 34, 72 ] } }, "spans": { "lab": { "score": 64, "band": "clear", "coverage": 5, "range": [ 39, 78 ] }, "regulator": { "score": 65, "band": "clear", "coverage": 5, "range": [ 39, 79 ] }, "public": { "score": 62, "band": "clear", "coverage": 5, "range": [ 46, 71 ] }, "equal": { "score": 60, "band": "clear", "coverage": 5, "range": [ 38, 75 ] } }, "primary": { "lab": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "regulator": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "public": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "equal": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] } } }, "band": "conditional", "coverage": 8, "floor": "A", "what_moves": { "lab": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 58, "band": "conditional", "base": 54 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 54 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 54 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 59, "band": "clear", "base": 54 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 57, "band": "conditional", "base": 54 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 57, "band": "conditional", "base": 54 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 54 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 55, "band": "conditional", "base": 54 } ], "regulator": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 57, "band": "conditional", "base": 54 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 54 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 54 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 59, "band": "clear", "base": 54 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 59, "band": "conditional", "base": 54 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 57, "band": "conditional", "base": 54 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 54 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 54, "band": "conditional", "base": 54 } ], "public": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 64, "band": "conditional", "base": 57 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 61, "band": "conditional", "base": 57 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 61, "band": "conditional", "base": 57 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 59, "band": "clear", "base": 57 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 60, "band": "conditional", "base": 57 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 63, "band": "conditional", "base": 57 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 57 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 57 } ], "equal": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "G", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "A", "from_value": 1, "to_value": 2, "score": 56, "band": "clear", "base": 53 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 }, { "dimension": "X", "from_value": 2, "to_value": 3, "score": 56, "band": "conditional", "base": 53 } ] } }, { "id": "scale", "name": "Scale AI (SEAL / Scale Labs)", "type": "vc", "hq": "San Francisco, US", "domains": [ "benchmarks", "cyber", "bio", "misuse" ], "confidence": "high", "summary": "Builds and runs evaluations for CAISI and other governments; Meta acquired 49% in 2025.", "what_would_move_the_score": "Ownership separation of the evaluation unit; until then the ceiling is low.", "role": "vendor", "dissent": { "lower": "Personnel (1 to 0). The founder joined Meta as an executive under the $14.3B ownership transaction and, by Scale's own announcement, keeps his seat on Scale's board. No recusal is documented anywhere in the public record. P.7 routes leadership that moved to a lab under an ownership transaction to P.1, which gives 0 without a recusal; P.4 separately caps a lab employee on the evaluator's board at 2.", "higher": "Funding (0 to 1). Meta's 49% stake is non-voting, Scale's own announcement calls it a minority of outstanding equity, and the interim CEO stated Scale 'remains, unequivocally, an independent company'. If the 20% threshold in F.1 is read as requiring control rather than economic interest, the record supports anchor 1: a material investment from a frontier developer without voting power." }, "list_group": "commercial", "graph_id": "evaluator#226", "values": { "F": 0, "G": 0, "P": 0, "A": 3, "S": 2, "R": 2, "M": 2, "X": 1 }, "values_by_policy": { "leads": { "F": 0, "G": 0, "P": 0, "A": 3, "S": 2, "R": 2, "M": 2, "X": 1 }, "standard": { "F": 0, "G": 0, "P": 0, "A": 3, "S": 2, "R": 2, "M": 2, "X": 1 }, "against_interest": { "F": 0, "G": 0, "P": 0, "A": 3, "S": 2, "R": 2, "M": 2, "X": 1 }, "spans": { "F": 0, "G": 0, "P": 0, "A": 3, "S": 2, "R": 2, "M": 2, "X": 1 }, "primary": { "F": null, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "anchor": 0, "assessed": "2026-09-14", "assessor": "yohei/claude v0", "dimension": "F", "evaluator": "scale", "rationale": "Anchor 0 on funding given the cited signals. Curator disclosure (2026-09-15): the curator holds small public-market shareholdings in Google and Meta. This evaluator has a confirmed ledger tie to Meta (investment row T52). Per the disclosure Rule, this assessment is flagged for public review: the rationale names the holding so readers can weigh it, and any reader can file a correction through the contribution path.", "signals": [ "scale.01" ], "value": 0, "graph_id": "assessment#771", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 0, "b": [ "scale.01" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 0, "b": [ "scale.01" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 0, "b": [ "scale.01" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 0, "b": [ "scale.01" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "scale.01" ] } }, "rationale_derived": "Capped at 0 by scale.01 (F.1): Nearly half owned by Meta, itself a frontier developer..", "rationale_leads": "Capped at 0 by scale.01 (F.1): Nearly half owned by Meta, itself a frontier developer.." }, "G": { "anchor": 0, "assessed": "2026-09-15", "assessor": "yohei/claude v1.0", "dimension": "G", "evaluator": "scale", "signals": [ "scale.02", "scale.10" ], "value": 0, "graph_id": "assessment#772", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 0, "b": [ "scale.10" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 0, "b": [ "scale.10" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 0, "b": [ "scale.10" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 0, "b": [ "scale.10" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "scale.02", "scale.10" ] } }, "rationale_derived": "Capped at 0 by scale.10 (G.6): Meta holds a 49% stake after a $14.3B investment; the founder left for Meta and an interim CEO was named; Google, OpenAI and xAI reportedly reduced engagements..", "rationale_leads": "Capped at 0 by scale.10 (G.6): Meta holds a 49% stake after a $14.3B investment; the founder left for Meta and an interim CEO was named; Google, OpenAI and xAI reportedly reduced engagements.." }, "P": { "anchor": 0, "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "dimension": "P", "evaluator": "scale", "signals": [ "scale.05", "scale.11" ], "value": 0, "open_questions": [ "Request from Scale AI the board-recusal terms, if any, that apply to Alexandr Wang's continuing directorship while he is employed by Meta (Scale announcement, 12 June 2025), and whether he sits on any committee reviewing evaluation work. Per RULES 12 this goes to the person before publication." ], "graph_id": "assessment#773", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 0, "b": [ "scale.05" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 0, "b": [ "scale.05" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 0, "b": [ "scale.05" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 0, "b": [ "scale.05" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "scale.05", "scale.11" ] } }, "rationale_derived": "Capped at 0 by scale.05 (P.7 (via P.1)): Leadership moved to Meta with the 2025 investment..", "rationale_leads": "Capped at 0 by scale.05 (P.7 (via P.1)): Leadership moved to Meta with the 2025 investment.." }, "A": { "anchor": 3, "assessed": "2026-09-14", "assessor": "yohei/claude v0", "dimension": "A", "evaluator": "scale", "signals": [ "scale.03" ], "value": 3, "mechanism": "lab-controlled", "graph_id": "assessment#774", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "scale.03" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "scale.03" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "scale.03" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "scale.03" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "scale.03" ] } }, "rationale_derived": "Floored at 3 by scale.03 (A.4): Government-defined risk domains and access agreements; first private evaluator authorized by the US institute.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by scale.03 (A.4): Government-defined risk domains and access agreements; first private evaluator authorized by the US institute.. No admissible signal caps it." }, "S": { "anchor": 2, "assessed": "2026-09-14", "assessor": "yohei/claude v0", "dimension": "S", "evaluator": "scale", "signals": [ "scale.06" ], "value": 2, "graph_id": "assessment#775", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "scale.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "scale.06" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "scale.06" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "scale.06" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "scale.06" ] } }, "rationale_derived": "Capped at 2 by scale.06 (S.2): Scope set by government or lab clients..", "rationale_leads": "Capped at 2 by scale.06 (S.2): Scope set by government or lab clients.." }, "R": { "anchor": 2, "assessed": "2026-09-14", "assessor": "yohei/claude v0", "dimension": "R", "evaluator": "scale", "signals": [ "scale.07" ], "value": 2, "mechanism": "self-imposed", "graph_id": "assessment#776", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "scale.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "scale.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "scale.07" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "scale.07" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "scale.07" ] } }, "rationale_derived": "Capped at 2 by scale.07 (R.4 (by analogy)): Results for governments not public..", "rationale_leads": "Capped at 2 by scale.07 (R.4 (by analogy)): Results for governments not public.." }, "M": { "anchor": 2, "assessed": "2026-09-14", "assessor": "yohei/claude v0", "dimension": "M", "evaluator": "scale", "signals": [ "scale.08" ], "value": 2, "graph_id": "assessment#777", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "scale.08" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "scale.08" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "scale.08" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "scale.08" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "scale.08" ] } }, "rationale_derived": "Capped at 2 by scale.08 (M.2): Benchmarks partly public (SEAL leaderboards); government evaluations closed..", "rationale_leads": "Capped at 2 by scale.08 (M.2): Benchmarks partly public (SEAL leaderboards); government evaluations closed.." }, "X": { "anchor": 1, "assessed": "2026-09-14", "assessor": "yohei/claude v0", "dimension": "X", "evaluator": "scale", "signals": [ "scale.04" ], "value": 1, "graph_id": "assessment#778", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 1, "b": [ "scale.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "scale.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "scale.04" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 1, "b": [ "scale.04" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "scale.04" ] } }, "rationale_derived": "Capped at 1 by scale.04 (X.2): Sells data and evaluation services to labs..", "rationale_leads": "Capped at 1 by scale.04 (X.2): Sells data and evaluation services to labs.." } }, "signals": [ { "id": "scale.01", "evaluator": "scale", "dimension": "F", "direction": "against", "claim": "Nearly half owned by Meta, itself a frontier developer.", "sources": [ "longtermwiki-audit", "scale-framework", "cnbc-scale-meta", "scale-next-phase" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 0 }, "rule": "F.1", "quote": "Meta has a 49% stake in Scale after its $14.3 billion investment", "quote_source": "cnbc-scale-meta", "graph_id": "signal#527", "source_graph_ids": [ "source#104", "source#164", "source#50", "source#165" ] }, { "id": "scale.02", "evaluator": "scale", "dimension": "G", "direction": "against", "claim": "Primary business is selling data and evaluation services to the labs; no public COI policy addressing the Meta relationship.", "sources": [ "scale-framework" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 1 }, "rule": "G.2", "quote": "Scale has worked with the world's leading AI model builders", "quote_source": "scale-framework", "graph_id": "signal#528", "source_graph_ids": [ "source#164" ] }, { "id": "scale.03", "evaluator": "scale", "dimension": "A", "direction": "for", "claim": "Government-defined risk domains and access agreements; first private evaluator authorized by the US institute.", "sources": [ "longtermwiki-audit", "scale-framework" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "A.4", "quote": "Scale AI SEAL : Safety, Evaluation, and Alignment Lab; first third-party evaluator authorized by US AISI", "quote_source": "longtermwiki-audit", "graph_id": "signal#529", "source_graph_ids": [ "source#104", "source#164" ] }, { "id": "scale.04", "evaluator": "scale", "dimension": "X", "direction": "against", "claim": "Sells data and evaluation services to labs.", "sources": [ "scale-framework", "cnbc-scale-meta" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 1 }, "rule": "X.2", "quote": "Meta has been one of Scale AI's biggest customers", "quote_source": "cnbc-scale-meta", "graph_id": "signal#530", "source_graph_ids": [ "source#164", "source#50" ] }, { "id": "scale.05", "evaluator": "scale", "dimension": "P", "direction": "against", "claim": "Leadership moved to Meta with the 2025 investment.", "sources": [ "longtermwiki-audit", "cnbc-scale-meta", "scale-next-phase" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 0 }, "rule": "P.7 (via P.1)", "quote": "a deal that included the departure of Scale AI founder Alexandr Wang for the social media company", "quote_source": "cnbc-scale-meta", "graph_id": "signal#531", "source_graph_ids": [ "source#104", "source#50", "source#165" ] }, { "id": "scale.06", "evaluator": "scale", "dimension": "S", "direction": "against", "claim": "Scope set by government or lab clients.", "sources": [ "scale-framework" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "S.2", "quote": "The government defines the risk domains it wants evaluated", "quote_source": "scale-framework", "graph_id": "signal#532", "source_graph_ids": [ "source#164" ] }, { "id": "scale.07", "evaluator": "scale", "dimension": "R", "direction": "against", "claim": "Results for governments not public.", "sources": [ "scale-framework" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "R.4 (by analogy)", "quote": "The government provides subject matter expertise and interprets the results for policy and standards", "quote_source": "scale-framework", "graph_id": "signal#533", "source_graph_ids": [ "source#164" ] }, { "id": "scale.08", "evaluator": "scale", "dimension": "M", "direction": "against", "claim": "Benchmarks partly public (SEAL leaderboards); government evaluations closed.", "sources": [ "scale-framework" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "M.2", "quote": "Scale Labs builds the benchmarks and conducts the technical testing", "quote_source": "scale-framework", "graph_id": "signal#534", "source_graph_ids": [ "source#164" ] }, { "id": "scale.09", "evaluator": "scale", "dimension": "F", "direction": "against", "claim": "Meta paid $14.3B for a 49% non-voting stake in June 2025; the founder moved to Meta.", "sources": [ "evaluators-ledger", "cnbc-scale-meta" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-11", "bound": { "cap": 0 }, "rule": "F.1", "quote": "Meta has a 49% stake in Scale after its $14.3 billion investment", "quote_source": "cnbc-scale-meta", "graph_id": "signal#535", "source_graph_ids": [ "source#81", "source#50" ] }, { "id": "scale.10", "evaluator": "scale", "dimension": "G", "direction": "against", "claim": "Meta holds a 49% stake after a $14.3B investment; the founder left for Meta and an interim CEO was named; Google, OpenAI and xAI reportedly reduced engagements.", "sources": [ "cnbc-scale-meta" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2025-06-18", "quote": "Meta has a 49% stake in Scale AI after investing $14.3 billion", "bound": { "cap": 0 }, "rule": "G.6", "quote_source": "cnbc-scale-meta", "graph_id": "signal#536", "source_graph_ids": [ "source#50" ] }, { "id": "scale.11", "evaluator": "scale", "dimension": "P", "direction": "against", "claim": "After joining Meta, the founder continues to serve as a director on Scale's board, per Scale's own announcement of the investment.", "sources": [ "scale-next-phase" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 2 }, "rule": "P.4", "quote": "Wang will continue to serve as a director on the Scale Board of Directors", "quote_source": "scale-next-phase", "note": "Needed to apply P.4: a board member who is an employee of a lab caps at 2 regardless of recusal. Admission on the organization's own page.", "graph_id": "signal#537", "source_graph_ids": [ "source#165" ] } ], "scores": { "lab": 36, "regulator": 38, "public": 23, "equal": 31 }, "scores_by_policy": { "leads": { "lab": { "score": 36, "band": "disqualifying", "coverage": 8, "range": [ 36, 36 ] }, "regulator": { "score": 38, "band": "disqualifying", "coverage": 8, "range": [ 38, 38 ] }, "public": { "score": 23, "band": "disqualifying", "coverage": 8, "range": [ 23, 23 ] }, "equal": { "score": 31, "band": "disqualifying", "coverage": 8, "range": [ 31, 31 ] } }, "standard": { "lab": { "score": 36, "band": "disqualifying", "coverage": 8, "range": [ 36, 36 ] }, "regulator": { "score": 38, "band": "disqualifying", "coverage": 8, "range": [ 38, 38 ] }, "public": { "score": 23, "band": "disqualifying", "coverage": 8, "range": [ 23, 23 ] }, "equal": { "score": 31, "band": "disqualifying", "coverage": 8, "range": [ 31, 31 ] } }, "against_interest": { "lab": { "score": 36, "band": "disqualifying", "coverage": 8, "range": [ 36, 36 ] }, "regulator": { "score": 38, "band": "disqualifying", "coverage": 8, "range": [ 38, 38 ] }, "public": { "score": 23, "band": "disqualifying", "coverage": 8, "range": [ 23, 23 ] }, "equal": { "score": 31, "band": "disqualifying", "coverage": 8, "range": [ 31, 31 ] } }, "spans": { "lab": { "score": 36, "band": "disqualifying", "coverage": 8, "range": [ 36, 36 ] }, "regulator": { "score": 38, "band": "disqualifying", "coverage": 8, "range": [ 38, 38 ] }, "public": { "score": 23, "band": "disqualifying", "coverage": 8, "range": [ 23, 23 ] }, "equal": { "score": 31, "band": "disqualifying", "coverage": 8, "range": [ 31, 31 ] } }, "primary": { "lab": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "regulator": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "public": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "equal": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] } } }, "band": "disqualifying", "coverage": 8, "floor": "F", "what_moves": { "lab": [ { "dimension": "F", "from_value": 0, "to_value": 1, "score": 40, "band": "disqualifying", "base": 36 }, { "dimension": "G", "from_value": 0, "to_value": 1, "score": 38, "band": "disqualifying", "base": 36 }, { "dimension": "P", "from_value": 0, "to_value": 1, "score": 38, "band": "disqualifying", "base": 36 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 41, "band": "disqualifying", "base": 36 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 40, "band": "disqualifying", "base": 36 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 39, "band": "disqualifying", "base": 36 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 38, "band": "disqualifying", "base": 36 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 37, "band": "disqualifying", "base": 36 } ], "regulator": [ { "dimension": "F", "from_value": 0, "to_value": 1, "score": 41, "band": "disqualifying", "base": 38 }, { "dimension": "G", "from_value": 0, "to_value": 1, "score": 40, "band": "disqualifying", "base": 38 }, { "dimension": "P", "from_value": 0, "to_value": 1, "score": 40, "band": "disqualifying", "base": 38 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 43, "band": "disqualifying", "base": 38 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 43, "band": "disqualifying", "base": 38 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 41, "band": "disqualifying", "base": 38 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 40, "band": "disqualifying", "base": 38 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 38, "band": "disqualifying", "base": 38 } ], "public": [ { "dimension": "F", "from_value": 0, "to_value": 1, "score": 29, "band": "disqualifying", "base": 23 }, { "dimension": "G", "from_value": 0, "to_value": 1, "score": 26, "band": "disqualifying", "base": 23 }, { "dimension": "P", "from_value": 0, "to_value": 1, "score": 26, "band": "disqualifying", "base": 23 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 24, "band": "disqualifying", "base": 23 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 25, "band": "disqualifying", "base": 23 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 28, "band": "disqualifying", "base": 23 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 24, "band": "disqualifying", "base": 23 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 24, "band": "disqualifying", "base": 23 } ], "equal": [ { "dimension": "F", "from_value": 0, "to_value": 1, "score": 34, "band": "disqualifying", "base": 31 }, { "dimension": "G", "from_value": 0, "to_value": 1, "score": 34, "band": "disqualifying", "base": 31 }, { "dimension": "P", "from_value": 0, "to_value": 1, "score": 34, "band": "disqualifying", "base": 31 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 34, "band": "disqualifying", "base": 31 }, { "dimension": "S", "from_value": 2, "to_value": 3, "score": 34, "band": "disqualifying", "base": 31 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 34, "band": "disqualifying", "base": 31 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 34, "band": "disqualifying", "base": 31 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 34, "band": "disqualifying", "base": 31 } ] } }, { "id": "securebio", "name": "SecureBio", "type": "nonprofit", "hq": "Cambridge, US", "domains": [ "bio", "misuse" ], "confidence": "high", "summary": "Main biosecurity evaluator; author of the Virology Capabilities Test; founding member of the AI Evaluator Forum.", "what_would_move_the_score": "Publish the AI division's client mix and the written terms that insulate it from Detection's funders.", "role": "vendor", "dissent": { "lower": "Funding should be 0. The OpenAI Foundation controls OpenAI Group (26% stake, appoints the entire board) and its $17.2M grant exceeds SecureBio's whole 2024 revenue; OpenAI PBC also pays for the evaluations of its own models. Money from a lab's controlling parent plus fees from the lab, with SecureBio's largest division dependent on it, is closer to 'controlled by a frontier developer' than to anchor 1.", "higher": "Funding should be 2. The grant is restricted to Detection, the AI team keeps separate leadership and budgets, the COI policy caps AI-company services at 25% of revenue and bars results-contingent fees, and the FY2024 990 shows 93% contributions with 4.5% program services. The Foundation is a nonprofit grantmaker, not a return-seeking investor; F.3's per-engagement fees, disclosed per report, put this at 2." }, "list_group": "commercial", "graph_id": "evaluator#227", "values": { "F": 1, "G": 3, "P": 3, "A": 2, "S": 3, "R": 3, "M": 2, "X": 1 }, "values_by_policy": { "leads": { "F": 1, "G": 3, "P": 3, "A": 2, "S": 3, "R": 3, "M": 2, "X": 1 }, "standard": { "F": 1, "G": 3, "P": 3, "A": 2, "S": 3, "R": 3, "M": 2, "X": 1 }, "against_interest": { "F": 1, "G": 2, "P": 2, "A": null, "S": 3, "R": 2, "M": 2, "X": 1 }, "spans": { "F": 1, "G": 3, "P": 3, "A": 2, "S": 3, "R": 3, "M": 2, "X": 1 }, "primary": { "F": 1, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "securebio", "dimension": "F", "value": 1, "anchor": 1, "signals": [ "securebio.01", "securebio.02", "securebio.11", "securebio.13", "securebio.16", "securebio.22", "securebio.23" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "open_questions": [ "Coefficient grant amount to SecureBio." ], "resolution": { "rule": "F.5", "value": 1, "note": "F.5 (material revenue from a lab investor) decides over the F.9 public-contract floor; the floor describes the remainder of the base." }, "graph_id": "assessment#779", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 1, "b": [ "securebio.02", "securebio.11" ], "h": false, "c": true, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "securebio.02", "securebio.11" ], "h": false, "c": true, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "securebio.02", "securebio.11" ], "h": false, "c": false, "u": false, "x": [ "securebio.13", "securebio.23" ] }, "spans": { "v": 1, "b": [ "securebio.02", "securebio.11" ], "h": false, "c": true, "u": false, "x": [] }, "primary": { "v": 1, "b": [ "securebio.02", "securebio.11" ], "h": false, "c": false, "u": false, "x": [ "securebio.01", "securebio.13", "securebio.16", "securebio.22", "securebio.23" ] } }, "rationale_derived": "Conflict: floor 3 from securebio.13 (F.9) against cap 1 from securebio.02, securebio.11 (F.5, F.5); resolved at 1 by F.5: F.5 (material revenue from a lab investor) decides over the F.9 public-contract floor; the floor describes the remainder of the base.", "rationale_leads": "Conflict: floor 3 from securebio.13 (F.9) against cap 1 from securebio.02, securebio.11 (F.5, F.5); resolved at 1 by F.5: F.5 (material revenue from a lab investor) decides over the F.9 public-contract floor; the floor describes the remainder of the base." }, "G": { "evaluator": "securebio", "dimension": "G", "value": 3, "anchor": 3, "signals": [ "securebio.03", "securebio.04", "securebio.12", "securebio.17" ], "assessed": "2026-09-15", "assessor": "yohei/claude v1.0", "graph_id": "assessment#780", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "securebio.17" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "securebio.17" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "securebio.04" ], "h": false, "c": false, "u": false, "x": [ "securebio.03", "securebio.12", "securebio.17" ] }, "spans": { "v": 3, "b": [ "securebio.17" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "securebio.03", "securebio.04", "securebio.12", "securebio.17" ] } }, "rationale_derived": "Floored at 3 by securebio.17 (G.2): SecureBio publishes a conflicts-of-interest policy requiring recusal for financial interests or recent employment at the assessed entity and barring results-contingent funding.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by securebio.17 (G.2): SecureBio publishes a conflicts-of-interest policy requiring recusal for financial interests or recent employment at the assessed entity and barring results-contingent funding.. No admissible signal caps it." }, "P": { "evaluator": "securebio", "dimension": "P", "value": 3, "anchor": 3, "signals": [ "securebio.08", "securebio.19" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#781", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "securebio.19" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "securebio.19" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "securebio.08" ], "h": false, "c": false, "u": false, "x": [ "securebio.19" ] }, "spans": { "v": 3, "b": [ "securebio.19" ], "h": false, "c": false, "u": false, "x": [ "securebio.08" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "securebio.08", "securebio.19" ] } }, "rationale_derived": "Floored at 3 by securebio.19 (P.5): SecureBio's COI policy makes disclosure of conflicts mandatory for all AI-team staff at hiring and every six months, requires recusal, and its reports disclose conflicts relevant to each assessment.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by securebio.19 (P.5): SecureBio's COI policy makes disclosure of conflicts mandatory for all AI-team staff at hiring and every six months, requires recusal, and its reports disclose conflicts relevant to each assessment.. No admissible signal caps it." }, "A": { "evaluator": "securebio", "dimension": "A", "value": 2, "anchor": 2, "signals": [ "securebio.07", "securebio.20" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "lab-controlled", "graph_id": "assessment#782", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "securebio.20" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "securebio.20" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "securebio.07", "securebio.20" ] }, "spans": { "v": 2, "b": [ "securebio.20" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "securebio.07", "securebio.20" ] } }, "rationale_derived": "Floored at 2 by securebio.20 (A.1): For the GPT-5.5 pre-release assessment, OpenAI disabled API-level biological content filtering on the checkpoints SecureBio tested.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by securebio.20 (A.1): For the GPT-5.5 pre-release assessment, OpenAI disabled API-level biological content filtering on the checkpoints SecureBio tested.. No admissible signal caps it." }, "S": { "evaluator": "securebio", "dimension": "S", "value": 3, "anchor": 3, "signals": [ "securebio.05", "securebio.14", "securebio.21" ], "assessed": "2026-09-15", "assessor": "yohei/claude v0-final", "graph_id": "assessment#783", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "securebio.21" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "securebio.21" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "securebio.21" ], "h": false, "c": false, "u": false, "x": [ "securebio.14" ] }, "spans": { "v": 3, "b": [ "securebio.21" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "securebio.05", "securebio.14", "securebio.21" ] } }, "rationale_derived": "Capped at 3 by securebio.21 (S.1): The GPT-5.5 pre-release window was set by the lab: access ran from April 2 to April 9, 2026.. Floors up to 3 from securebio.05 do not exceed the cap.", "rationale_leads": "Capped at 3 by securebio.21 (S.1): The GPT-5.5 pre-release window was set by the lab: access ran from April 2 to April 9, 2026.. Floors up to 3 from securebio.05 do not exceed the cap." }, "R": { "evaluator": "securebio", "dimension": "R", "value": 3, "anchor": 3, "signals": [ "securebio.06", "securebio.18" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "mechanism": "self-imposed", "graph_id": "assessment#784", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "securebio.18" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "securebio.18" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "securebio.06" ], "h": false, "c": false, "u": false, "x": [ "securebio.18" ] }, "spans": { "v": 3, "b": [ "securebio.18" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "securebio.06", "securebio.18" ] } }, "rationale_derived": "Floored at 3 by securebio.18 (R.1): SecureBio's principles state developers have no authority to redact results to conceal unfavorable findings, that redactions are limited to infohazards and confidential business information, and that reports say whether redactions affect conclusions.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by securebio.18 (R.1): SecureBio's principles state developers have no authority to redact results to conceal unfavorable findings, that redactions are limited to infohazards and confidential business information, and that reports say whether redactions affect conclusions.. No admissible signal caps it." }, "M": { "evaluator": "securebio", "dimension": "M", "value": 2, "anchor": 2, "signals": [ "securebio.09" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#785", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "securebio.09" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "securebio.09" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "securebio.09" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "securebio.09" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "securebio.09" ] } }, "rationale_derived": "Floored at 2 by securebio.09 (M.2): Methods published; VCT is documented.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by securebio.09 (M.2): Methods published; VCT is documented.. No admissible signal caps it." }, "X": { "evaluator": "securebio", "dimension": "X", "value": 1, "anchor": 1, "signals": [ "securebio.10", "securebio.15" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "resolution": { "rule": "X.2", "value": 1, "note": "securebio.10's raw floor of 4 is held to 3 by C15/C16 and still exceeds the X.2 cap; X.2 decides because the licensing statement is on SecureBio's own page." }, "graph_id": "assessment#786", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 1, "b": [ "securebio.15" ], "h": true, "c": true, "u": false, "x": [] }, "standard": { "v": 1, "b": [ "securebio.15" ], "h": true, "c": true, "u": false, "x": [] }, "against_interest": { "v": 1, "b": [ "securebio.15" ], "h": false, "c": false, "u": false, "x": [ "securebio.10" ] }, "spans": { "v": 1, "b": [ "securebio.15" ], "h": false, "c": false, "u": false, "x": [ "securebio.10" ] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "securebio.10", "securebio.15" ] } }, "rationale_derived": "Conflict: floor 3 from securebio.10 (X.7) against cap 1 from securebio.15 (X.2); resolved at 1 by X.2: securebio.10's raw floor of 4 is held to 3 by C15/C16 and still exceeds the X.2 cap; X.2 decides because the licensing statement is on SecureBio's own page.", "rationale_leads": "Conflict: floor 3 from securebio.10 (X.7) against cap 1 from securebio.15 (X.2); resolved at 1 by X.2: securebio.10's raw floor of 4 is held to 3 by C15/C16 and still exceeds the X.2 cap; X.2 decides because the licensing statement is on SecureBio's own page." } }, "signals": [ { "id": "securebio.01", "evaluator": "securebio", "dimension": "F", "direction": "against", "claim": "Labs typically pay for evaluations of their own models; OpenAI covered the cost of the GPT-5.5 assessment.", "sources": [ "securebio-oaif", "securebio-eaforum-oaif", "securebio-principles", "securebio-coi-policy" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "quote": "OpenAI PBC covered SecureBio's costs for the GPT 5.5 evaluation", "bound": { "cap": 2 }, "rule": "F.3", "quote_source": "securebio-eaforum-oaif", "graph_id": "signal#538", "source_graph_ids": [ "source#171", "source#169", "source#172", "source#168" ] }, { "id": "securebio.02", "evaluator": "securebio", "dimension": "F", "direction": "against", "claim": "Accepted an OpenAI Foundation grant in 2026 for the Detection division; the foundation's endowment is a stake in OpenAI and its board largely overlaps.", "sources": [ "securebio-oaif", "cnbc-openai-foundation", "openai-structure", "propublica-securebio-990" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 1 }, "rule": "F.5", "quote": "the OpenAI Foundation will hold a 26% stake in the for-profit", "quote_source": "cnbc-openai-foundation", "graph_id": "signal#539", "source_graph_ids": [ "source#171", "source#49", "source#144", "source#156" ] }, { "id": "securebio.03", "evaluator": "securebio", "dimension": "G", "direction": "for", "claim": "Two-division structure with a stated firewall; leadership committed publicly to resign and disclose if the grant were used as leverage on evaluations.", "sources": [ "securebio-oaif", "securebio-eaforum-oaif" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "G.1", "quote": "I'll say so publicly and use whatever leverage I have to stop it (up to and including resignation)", "quote_source": "securebio-eaforum-oaif", "graph_id": "signal#540", "source_graph_ids": [ "source#171", "source#169" ] }, { "id": "securebio.04", "evaluator": "securebio", "dimension": "G", "direction": "for", "claim": "Founding member of the AI Evaluator Forum, which wrote the AEF-1 independence standard.", "sources": [ "aef-launch", "securebio-principles" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "G.1", "quote": "As members of the AI Evaluator Forum (AEF), we co-developed and have adopted the AEF-1 Standard", "quote_source": "securebio-principles", "graph_id": "signal#541", "source_graph_ids": [ "source#10", "source#172" ] }, { "id": "securebio.05", "evaluator": "securebio", "dimension": "S", "direction": "for", "claim": "Evaluates models from many developers; consortium member on the EU AI Office CBRN contract.", "sources": [ "longtermwiki-far" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "S.3", "quote": "FAR AI leads a consortium including SecureBio", "quote_source": "longtermwiki-far", "graph_id": "signal#542", "source_graph_ids": [ "source#106" ] }, { "id": "securebio.06", "evaluator": "securebio", "dimension": "R", "direction": "for", "claim": "Publishes pre-release biological assessments and the Virology Capabilities Test.", "sources": [ "securebio-oaif", "aef-launch" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "R.1", "quote": "SecureBio’s model evaluations, including pre-release assessments for biosecurity risk, are under our AI team", "quote_source": "securebio-oaif", "graph_id": "signal#543", "source_graph_ids": [ "source#171", "source#10" ] }, { "id": "securebio.07", "evaluator": "securebio", "dimension": "A", "direction": "for", "claim": "Pre-deployment access across the major Western labs.", "sources": [ "securebio-oaif" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 1 }, "rule": "A.1", "quote": "including pre-release assessments for biosecurity risk", "quote_source": "securebio-oaif", "graph_id": "signal#544", "source_graph_ids": [ "source#171" ] }, { "id": "securebio.08", "evaluator": "securebio", "dimension": "P", "direction": "for", "claim": "Staff pipeline disclosed through philanthropic career grants; no lab board roles found.", "sources": [ "80k-funding" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "P.6", "graph_id": "signal#545", "source_graph_ids": [ "source#9" ] }, { "id": "securebio.09", "evaluator": "securebio", "dimension": "M", "direction": "for", "claim": "Methods published; VCT is documented.", "sources": [ "aef-launch", "securebio-gpt55-assessment" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "M.2", "quote": "VCT comprises 322 questions on fundamental, tacit, and visual knowledge", "quote_source": "securebio-gpt55-assessment", "graph_id": "signal#546", "source_graph_ids": [ "source#10", "source#170" ] }, { "id": "securebio.10", "evaluator": "securebio", "dimension": "X", "direction": "for", "claim": "No commercial products found; tools shared with governments.", "sources": [ "securebio-oaif" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 4 }, "rule": "X.7", "graph_id": "signal#547", "source_graph_ids": [ "source#171" ] }, { "id": "securebio.11", "evaluator": "securebio", "dimension": "F", "direction": "against", "claim": "The OpenAI Foundation granted SecureBio Detection $17.2M; the Foundation's endowment is a stake of roughly a quarter of OpenAI PBC with a near-identical board.", "sources": [ "securebio-oaif", "securebio-x-oaif", "cnbc-openai-foundation", "propublica-securebio-990" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2026-09-15", "quote": "The OpenAI Foundation has granted SecureBio Detection $17.2M", "bound": { "cap": 1 }, "rule": "F.5", "quote_source": "securebio-oaif", "graph_id": "signal#548", "source_graph_ids": [ "source#171", "source#174", "source#49", "source#156" ] }, { "id": "securebio.12", "evaluator": "securebio", "dimension": "G", "direction": "for", "claim": "The grant is restricted to Detection work; the AI evaluation team has separate leadership, budgets and deliverables.", "sources": [ "securebio-oaif", "securebio-substack-detection" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2026-09-15", "quote": "This grant is restricted to where it may only fund our Detection work", "bound": { "floor": 2 }, "rule": "G.1", "quote_source": "securebio-oaif", "graph_id": "signal#549", "source_graph_ids": [ "source#171", "source#173" ] }, { "id": "securebio.13", "evaluator": "securebio", "dimension": "F", "direction": "for", "claim": "Coefficient multi-year grant and AI Safety Fund grants; US CAISI Bio R&D contract and EU AI Office contract in 2025.", "sources": [ "securebio-2025-review" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2026-01", "quote": "broadened our government-facing portfolio with a US CAISI Bio R&D contract", "bound": { "floor": 3 }, "rule": "F.9", "quote_source": "securebio-2025-review", "graph_id": "signal#550", "source_graph_ids": [ "source#166" ] }, { "id": "securebio.14", "evaluator": "securebio", "dimension": "S", "direction": "for", "claim": "SecureBio states the OpenAI Foundation grant places no constraints on what any part of SecureBio can research, publish or say, and its AI team's policy is to evaluate models solely on their merits.", "sources": [ "securebio-oaif" ], "recorded": "2026-09-15", "curator": "yohei/claude v0-final", "as_of": "2026-08-20", "quote": "it places no constraints on what any part of SecureBio can research, publish, or say", "bound": { "floor": 2 }, "rule": "S.5", "quote_source": "securebio-oaif", "graph_id": "signal#551", "source_graph_ids": [ "source#171" ] }, { "id": "securebio.15", "evaluator": "securebio", "dimension": "X", "direction": "against", "claim": "SecureBio reports licensing its evaluations to multiple frontier labs as a revenue model and delivering pretraining data-filtering datasets and topic lists used in the design of several frontier models.", "sources": [ "securebio-2025-review", "securebio-coi-policy" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 1 }, "rule": "X.2", "quote": "licensing evaluations to multiple frontier labs", "quote_source": "securebio-2025-review", "note": "Evaluation products delivered to evaluated labs for money: 1 (X.2). Rule tension: section 10 treats fees for the evaluation itself as a funding matter, but licensing an evaluation suite for labs to run is a product sale. X.1 (0) is not applied: the COI policy says paid services cover evaluations and audits only, so the mitigation datasets are not shown to have been sold; document request on whether the data-filtering deliverables were paid. Also on the page: 'We delivered training datasets and lists of dangerous biological topics for pretraining data filtering'.", "graph_id": "signal#552", "source_graph_ids": [ "source#166", "source#168" ] }, { "id": "securebio.16", "evaluator": "securebio", "dimension": "F", "direction": "against", "claim": "SecureBio received multiple grants from the Frontier Model Forum's AI Safety Fund (written 'Foundation Model Forum' on the page), a pool funded by Anthropic, Google, Microsoft and OpenAI.", "sources": [ "securebio-2025-review" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "F.6", "quote": "We also received multiple grants from the Foundation Model Forum", "quote_source": "securebio-2025-review", "note": "Lab-linked pooled funds cap at 3 (anchor 3 text; F.6 by analogy since the pool's donors are public and are the labs themselves). Amounts undisclosed.", "graph_id": "signal#553", "source_graph_ids": [ "source#166" ] }, { "id": "securebio.17", "evaluator": "securebio", "dimension": "G", "direction": "for", "claim": "SecureBio publishes a conflicts-of-interest policy requiring recusal for financial interests or recent employment at the assessed entity and barring results-contingent funding.", "sources": [ "securebio-coi-policy", "securebio-principles" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "floor": 3 }, "rule": "G.2", "quote": "Staff will be required to recuse themselves from engagements where an actual or perceived conflict of interest exists", "quote_source": "securebio-coi-policy", "note": "Names at least two G.2 elements: recusal for financial interests and no outcome-contingent fees ('We only enter into engagements for which funding is not contingent on the results of our assessments'), plus disclosure of funding per report on the principles page. Nonprofit with a published policy: 3.", "graph_id": "signal#554", "source_graph_ids": [ "source#168", "source#172" ] }, { "id": "securebio.18", "evaluator": "securebio", "dimension": "R", "direction": "for", "claim": "SecureBio's principles state developers have no authority to redact results to conceal unfavorable findings, that redactions are limited to infohazards and confidential business information, and that reports say whether redactions affect conclusions.", "sources": [ "securebio-principles" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "floor": 3 }, "rule": "R.1", "quote": "AI developers have no authority to redact our results to conceal performance or unfavorable findings", "quote_source": "securebio-principles", "note": "Security-only redactions with a public statement: 3 (R.1). A 4 is unavailable under R.5 because reports are shared with developers before publication ('We share our reports with AI developers in advance of our publication'), even though the Opus 4.6 review records minor disagreements with Anthropic.", "graph_id": "signal#555", "source_graph_ids": [ "source#172" ] }, { "id": "securebio.19", "evaluator": "securebio", "dimension": "P", "direction": "for", "claim": "SecureBio's COI policy makes disclosure of conflicts mandatory for all AI-team staff at hiring and every six months, requires recusal, and its reports disclose conflicts relevant to each assessment.", "sources": [ "securebio-coi-policy", "securebio-principles" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "floor": 3 }, "rule": "P.5", "quote": "All staff and contractors on our AI team must disclose potential conflicts of interest", "quote_source": "securebio-coi-policy", "note": "Published mandatory recusal rule plus disclosure of relevant ties in public reports ('In addition to funding, we disclose other potential conflicts of interest relevant to the assessment'): 3 (P.5). Disclosure is internal to leadership and per-report, not a public register of individual ties; a 4 would need cooling-off and no-equity evidence at tier 1.", "graph_id": "signal#556", "source_graph_ids": [ "source#168", "source#172" ] }, { "id": "securebio.20", "evaluator": "securebio", "dimension": "A", "direction": "for", "claim": "For the GPT-5.5 pre-release assessment, OpenAI disabled API-level biological content filtering on the checkpoints SecureBio tested.", "sources": [ "securebio-gpt55-assessment" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "floor": 2 }, "rule": "A.1", "quote": "API-level biological content filtering was disabled on these checkpoints", "quote_source": "securebio-gpt55-assessment", "note": "Pre-release access with safeguards off: anchor 2 (A.1). The Opus 4.6 review adds access to an unredacted risk report and 110 pages of materials, which is document access, not model access, so it is noted rather than scored.", "graph_id": "signal#557", "source_graph_ids": [ "source#170" ] }, { "id": "securebio.21", "evaluator": "securebio", "dimension": "S", "direction": "against", "claim": "The GPT-5.5 pre-release window was set by the lab: access ran from April 2 to April 9, 2026.", "sources": [ "securebio-gpt55-assessment" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "S.1", "quote": "We had access to these checkpoints from April 2nd through April 9th, 2026", "quote_source": "securebio-gpt55-assessment", "note": "A lab-set window in the most recent major engagement caps at 3 (S.1). Not binding today; rule-required.", "graph_id": "signal#558", "source_graph_ids": [ "source#170" ] }, { "id": "securebio.22", "evaluator": "securebio", "dimension": "F", "direction": "against", "claim": "SecureBio's COI policy states that AI firms, including those whose models it assesses, provide it with free API credits, in undisclosed amounts.", "sources": [ "securebio-coi-policy" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "F.2", "quote": "AI firms, including those whose models we assess, provide us with free API credits", "quote_source": "securebio-coi-policy", "note": "Rule gap: F.2 covers disclosed in-kind and undisclosed in-kind the evaluator says it depends on; undisclosed credits with no dependency statement fall between. By anchor text 'no lab money' (4) is excluded, so cap 3; not binding.", "graph_id": "signal#559", "source_graph_ids": [ "source#168" ] }, { "id": "securebio.23", "evaluator": "securebio", "dimension": "F", "direction": "for", "claim": "SecureBio's external review of Anthropic's Opus 4.6 risk report was unpaid: 'SecureBio did not receive funding from Anthropic for this work'.", "sources": [ "securebio-anthropic-review" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "floor": 2 }, "rule": "F.3", "quote": "SecureBio did not receive funding from Anthropic for this work", "quote_source": "securebio-anthropic-review", "note": "An unpaid engagement beside paid ones; supports 'otherwise diversified' (anchor 2) and no more. Recorded so the fee pattern is not overstated.", "graph_id": "signal#560", "source_graph_ids": [ "source#167" ] } ], "scores": { "lab": 56, "regulator": 60, "public": 57, "equal": 56 }, "scores_by_policy": { "leads": { "lab": { "score": 56, "band": "conditional", "coverage": 8, "range": [ 56, 56 ] }, "regulator": { "score": 60, "band": "conditional", "coverage": 8, "range": [ 60, 60 ] }, "public": { "score": 57, "band": "conditional", "coverage": 8, "range": [ 57, 57 ] }, "equal": { "score": 56, "band": "conditional", "coverage": 8, "range": [ 56, 56 ] } }, "standard": { "lab": { "score": 56, "band": "conditional", "coverage": 8, "range": [ 56, 56 ] }, "regulator": { "score": 60, "band": "conditional", "coverage": 8, "range": [ 60, 60 ] }, "public": { "score": 57, "band": "conditional", "coverage": 8, "range": [ 57, 57 ] }, "equal": { "score": 56, "band": "conditional", "coverage": 8, "range": [ 56, 56 ] } }, "against_interest": { "lab": { "score": 48, "band": "conditional", "coverage": 7, "range": [ 38, 58 ] }, "regulator": { "score": 52, "band": "conditional", "coverage": 7, "range": [ 41, 61 ] }, "public": { "score": 45, "band": "conditional", "coverage": 7, "range": [ 43, 48 ] }, "equal": { "score": 46, "band": "conditional", "coverage": 7, "range": [ 41, 53 ] } }, "spans": { "lab": { "score": 56, "band": "conditional", "coverage": 8, "range": [ 56, 56 ] }, "regulator": { "score": 60, "band": "conditional", "coverage": 8, "range": [ 60, 60 ] }, "public": { "score": 57, "band": "conditional", "coverage": 8, "range": [ 57, 57 ] }, "equal": { "score": 56, "band": "conditional", "coverage": 8, "range": [ 56, 56 ] } }, "primary": { "lab": { "score": 25, "band": "conditional", "coverage": 1, "range": [ 5, 87 ] }, "regulator": { "score": 25, "band": "conditional", "coverage": 1, "range": [ 4, 89 ] }, "public": { "score": 25, "band": "conditional", "coverage": 1, "range": [ 6, 81 ] }, "equal": { "score": 25, "band": "conditional", "coverage": 1, "range": [ 3, 91 ] } } }, "band": "conditional", "coverage": 8, "floor": "F", "what_moves": { "lab": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 61, "band": "conditional", "base": 56 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 58, "band": "conditional", "base": 56 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 61, "band": "conditional", "base": 56 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 60, "band": "conditional", "base": 56 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 60, "band": "conditional", "base": 56 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 57, "band": "conditional", "base": 56 } ], "regulator": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 64, "band": "conditional", "base": 60 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 63, "band": "conditional", "base": 60 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 63, "band": "conditional", "base": 60 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 65, "band": "conditional", "base": 60 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 65, "band": "conditional", "base": 60 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 64, "band": "conditional", "base": 60 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 63, "band": "conditional", "base": 60 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 60, "band": "conditional", "base": 60 } ], "public": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 64, "band": "conditional", "base": 57 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 61, "band": "conditional", "base": 57 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 61, "band": "conditional", "base": 57 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 57 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 60, "band": "conditional", "base": 57 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 63, "band": "conditional", "base": 57 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 57 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 59, "band": "conditional", "base": 57 } ], "equal": [ { "dimension": "F", "from_value": 1, "to_value": 2, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "R", "from_value": 3, "to_value": 4, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "M", "from_value": 2, "to_value": 3, "score": 59, "band": "conditional", "base": 56 }, { "dimension": "X", "from_value": 1, "to_value": 2, "score": 59, "band": "conditional", "base": 56 } ] } }, { "id": "transluce", "name": "Transluce", "type": "nonprofit", "hq": "Berkeley, US", "domains": [ "misuse", "autonomy", "benchmarks", "assurance" ], "confidence": "high", "summary": "Nonprofit oversight-tools lab; convenes the AI Evaluator Forum that wrote AEF-1.", "what_would_move_the_score": "Routine pre-release access under AEF-1 terms.", "role": "referee", "dissent": { "lower": "Funding could be 1. In FY2025, 38% of Transluce's revenue came from the personal holdings of Anthropic and OpenAI employees, which are lab equity by another name; anchor 1 is material revenue or investment from evaluated labs or their investors, and 38% is material by any accounting test. The policy permits such donations, and the donors are not named or bounded in size.", "higher": "Funding could be 3. No revenue came from developers as organizations, no evaluation has been paid, the employee donations were unrestricted and are disclosed by share each year under a published policy aligned to AEF-1, and the remainder is philanthropy and government contracts including an EU AI Office consortium seat. Anchor 3 is mostly philanthropic or public money with some lab-linked funds, which describes 62% of the base." }, "list_group": "referee", "graph_id": "evaluator#228", "values": { "F": 2, "G": 3, "P": 2, "A": 2, "S": 3, "R": 2, "M": 3, "X": 3 }, "values_by_policy": { "leads": { "F": 2, "G": 3, "P": 2, "A": 2, "S": 3, "R": 2, "M": 3, "X": 3 }, "standard": { "F": 2, "G": 3, "P": 2, "A": 2, "S": 3, "R": 2, "M": 3, "X": 3 }, "against_interest": { "F": 2, "G": 2, "P": 2, "A": 2, "S": null, "R": 2, "M": null, "X": 3 }, "spans": { "F": 2, "G": 3, "P": 2, "A": 2, "S": 3, "R": 2, "M": 3, "X": 3 }, "primary": { "F": null, "G": null, "P": null, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "transluce", "dimension": "F", "value": 2, "anchor": 2, "signals": [ "transluce.03", "transluce.09" ], "rationale": "Anchor 2: no money from developers as organizations and no paid evaluations, but 38% of FY2025 revenue came from lab employees' personal holdings, which are lab equity. The disclosure is the best in the population; the exposure is still material.", "assessed": "2026-09-14", "assessor": "yohei/claude v0.4", "open_questions": [ "Whether employee donors are named or bounded in size.", "How the 38% moves in FY2026." ], "resolution": { "rule": "F.7", "value": 2, "note": "Transluce's disclosed FY2025 revenue includes 38% from the personal holdings of Anthropic and OpenAI employees, above the 10% threshold; F.7 caps at 2 despite the government and philanthropic base." }, "graph_id": "assessment#787", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "transluce.09" ], "h": false, "c": true, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "transluce.09" ], "h": false, "c": true, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "transluce.09" ], "h": false, "c": false, "u": false, "x": [ "transluce.03" ] }, "spans": { "v": 2, "b": [ "transluce.09" ], "h": false, "c": true, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "transluce.03", "transluce.09" ] } }, "rationale_derived": "Conflict: floor 3 from transluce.03 (F.12) against cap 2 from transluce.09 (F.7); resolved at 2 by F.7: Transluce's disclosed FY2025 revenue includes 38% from the personal holdings of Anthropic and OpenAI employees, above the 10% threshold; F.7 caps at 2 despite the government and philanthropic base.", "rationale_leads": "Conflict: floor 3 from transluce.03 (F.12) against cap 2 from transluce.09 (F.7); resolved at 2 by F.7: Transluce's disclosed FY2025 revenue includes 38% from the personal holdings of Anthropic and OpenAI employees, above the 10% threshold; F.7 caps at 2 despite the government and philanthropic base." }, "G": { "evaluator": "transluce", "dimension": "G", "value": 3, "anchor": 3, "signals": [ "transluce.01", "transluce.10" ], "rationale": "Anchor 3: nonprofit with a published COI policy and disclosure duties; board of three includes the CEO, so independent-board evidence is thin.", "assessed": "2026-09-14", "assessor": "yohei/claude v0.4", "graph_id": "assessment#788", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "transluce.10" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "transluce.10" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "transluce.01" ], "h": false, "c": false, "u": false, "x": [ "transluce.10" ] }, "spans": { "v": 3, "b": [ "transluce.10" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "transluce.01", "transluce.10" ] } }, "rationale_derived": "Floored at 3 by transluce.10 (G.2): Published independence policy aligned to AEF-1 2.3: annual disclosure of developer-linked revenue share, paid-evaluation disclosure, no organizational equity in developers, third-party ethics hotline.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by transluce.10 (G.2): Published independence policy aligned to AEF-1 2.3: annual disclosure of developer-linked revenue share, paid-evaluation disclosure, no organizational equity in developers, third-party ethics hotline.. No admissible signal caps it." }, "P": { "evaluator": "transluce", "dimension": "P", "value": 2, "anchor": 2, "signals": [ "transluce.04", "transluce.12" ], "rationale": "Anchor 2 (closest fit, provisional): the governance head previously led CAISI and no lab board roles were found (transluce.04), but a current advisor sits on Thinking Machines’ technical staff and a board member runs Halcyon Futures (transluce.12). Provisional readings are open to public correction through the contribution path.", "assessed": "2026-09-15", "assessor": "yohei/claude v0", "graph_id": "assessment#789", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "transluce.12" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "transluce.12" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "transluce.12" ], "h": false, "c": false, "u": false, "x": [ "transluce.04" ] }, "spans": { "v": 2, "b": [ "transluce.12" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "transluce.04", "transluce.12" ] } }, "rationale_derived": "Capped at 2 by transluce.12 (P.4): An advisor is on the technical staff of Thinking Machines; a board member runs Halcyon Futures, which funds AVERI and AIUC.. Floors up to 2 from transluce.04 do not exceed the cap.", "rationale_leads": "Capped at 2 by transluce.12 (P.4): An advisor is on the technical staff of Thinking Machines; a board member runs Halcyon Futures, which funds AVERI and AIUC.. Floors up to 2 from transluce.04 do not exceed the cap." }, "A": { "evaluator": "transluce", "dimension": "A", "value": 2, "anchor": 2, "signals": [ "transluce.06" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "mechanism": "lab-controlled", "graph_id": "assessment#790", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "transluce.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "transluce.06" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "transluce.06" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "transluce.06" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "transluce.06" ] } }, "rationale_derived": "Capped at 2 by transluce.06 (A.1): Mostly post-deployment access; used in Claude 4 pre-deployment analysis once rather than routinely..", "rationale_leads": "Capped at 2 by transluce.06 (A.1): Mostly post-deployment access; used in Claude 4 pre-deployment analysis once rather than routinely.." }, "S": { "evaluator": "transluce", "dimension": "S", "value": 3, "anchor": 3, "signals": [ "transluce.07" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#791", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "transluce.07" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "transluce.07" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "transluce.07" ] }, "spans": { "v": 3, "b": [ "transluce.07" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "transluce.07" ] } }, "rationale_derived": "Floored at 3 by transluce.07 (S.5): Sets its own evaluation questions.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by transluce.07 (S.5): Sets its own evaluation questions.. No admissible signal caps it." }, "R": { "evaluator": "transluce", "dimension": "R", "value": 2, "anchor": 2, "signals": [ "transluce.02" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "mechanism": "self-imposed", "graph_id": "assessment#792", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 2, "b": [ "transluce.02" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "transluce.02" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "transluce.02" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "transluce.02" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "transluce.02" ] } }, "rationale_derived": "Floored at 2 by transluce.02 (R.7): Published the largest independent cross-vendor evaluation of responses to mental-health crises (77 variants) without lab sign-off.. No admissible signal caps it.", "rationale_leads": "Floored at 2 by transluce.02 (R.7): Published the largest independent cross-vendor evaluation of responses to mental-health crises (77 variants) without lab sign-off.. No admissible signal caps it." }, "M": { "evaluator": "transluce", "dimension": "M", "value": 3, "anchor": 3, "signals": [ "transluce.08" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#793", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "transluce.08" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "transluce.08" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "transluce.08" ] }, "spans": { "v": 3, "b": [ "transluce.08" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "transluce.08" ] } }, "rationale_derived": "Floored at 3 by transluce.08 (M.1): Open tools (Docent) and published methods.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by transluce.08 (M.1): Open tools (Docent) and published methods.. No admissible signal caps it." }, "X": { "evaluator": "transluce", "dimension": "X", "value": 3, "anchor": 3, "signals": [ "transluce.05" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#794", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "transluce.05" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "transluce.05" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "transluce.05" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "transluce.05" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "transluce.05" ] } }, "rationale_derived": "Capped at 3 by transluce.05 (X.4): Docent is used inside Anthropic, DeepMind and Thinking Machines, so labs are also tool users..", "rationale_leads": "Capped at 3 by transluce.05 (X.4): Docent is used inside Anthropic, DeepMind and Thinking Machines, so labs are also tool users.." } }, "signals": [ { "id": "transluce.01", "evaluator": "transluce", "dimension": "G", "direction": "for", "claim": "Wrote the field's independence standard (AEF-1) through the AI Evaluator Forum and applies it to itself.", "sources": [ "aef-launch", "transluce-job", "aef-transparency" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "G.1", "quote": "Transluce organizes the   AI Evaluator Forum", "quote_source": "transluce-job", "graph_id": "signal#561", "source_graph_ids": [ "source#10", "source#190", "source#11" ] }, { "id": "transluce.02", "evaluator": "transluce", "dimension": "R", "direction": "for", "claim": "Published the largest independent cross-vendor evaluation of responses to mental-health crises (77 variants) without lab sign-off.", "sources": [ "transluce-mh", "axios-transluce-mh" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "R.7", "quote": "The most expansive independent evaluation to date of how leading AI models respond to users in mental health crises", "quote_source": "transluce-mh", "graph_id": "signal#562", "source_graph_ids": [ "source#192", "source#43" ] }, { "id": "transluce.03", "evaluator": "transluce", "dimension": "F", "direction": "for", "claim": "Government contracts and philanthropy rather than lab fees.", "sources": [ "transluce-job", "transluce-manifund" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "F.12", "quote": "We build this technology using philanthropic funding", "quote_source": "transluce-manifund", "graph_id": "signal#563", "source_graph_ids": [ "source#190", "source#191" ] }, { "id": "transluce.04", "evaluator": "transluce", "dimension": "P", "direction": "for", "claim": "Head of governance previously led CAISI; no lab board roles found.", "sources": [ "transluce-manifund" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 2 }, "rule": "P.6", "quote": "previously led the U.S. Center for Standards and Innovation", "quote_source": "transluce-manifund", "graph_id": "signal#564", "source_graph_ids": [ "source#191" ] }, { "id": "transluce.05", "evaluator": "transluce", "dimension": "X", "direction": "against", "claim": "Docent is used inside Anthropic, DeepMind and Thinking Machines, so labs are also tool users.", "sources": [ "transluce-manifund" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 3 }, "rule": "X.4", "quote": "has been used by over 25 organizations, including frontier AI labs (Anthropic, DeepMind, Thinking Machines", "quote_source": "transluce-manifund", "graph_id": "signal#565", "source_graph_ids": [ "source#191" ] }, { "id": "transluce.06", "evaluator": "transluce", "dimension": "A", "direction": "against", "claim": "Mostly post-deployment access; used in Claude 4 pre-deployment analysis once rather than routinely.", "sources": [ "transluce-manifund" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "A.1", "quote": "It was also used as part of Claude 4's pre-deployment safety analysis", "quote_source": "transluce-manifund", "graph_id": "signal#566", "source_graph_ids": [ "source#191" ] }, { "id": "transluce.07", "evaluator": "transluce", "dimension": "S", "direction": "for", "claim": "Sets its own evaluation questions.", "sources": [ "transluce-mh" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "S.5", "quote": "The most expansive independent evaluation to date of how leading AI models respond to users in mental health crises", "quote_source": "transluce-mh", "graph_id": "signal#567", "source_graph_ids": [ "source#192" ] }, { "id": "transluce.08", "evaluator": "transluce", "dimension": "M", "direction": "for", "claim": "Open tools (Docent) and published methods.", "sources": [ "transluce-manifund" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "M.1", "quote": "Our open-source, AI-backed tools provide a technology stack that powers this public accountability", "quote_source": "transluce-manifund", "graph_id": "signal#568", "source_graph_ids": [ "source#191" ] }, { "id": "transluce.09", "evaluator": "transluce", "dimension": "F", "direction": "against", "claim": "FY2025 revenue: 32% from personal holdings of Anthropic employees and 6% from OpenAI employees, unrestricted, disclosed under its own policy; no revenue from developers as organizations.", "sources": [ "transluce-policy" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "quote": "32% of its revenue from the personal holdings of Anthropic employees", "bound": { "cap": 2 }, "rule": "F.7", "quote_source": "transluce-policy", "graph_id": "signal#569", "source_graph_ids": [ "source#193" ] }, { "id": "transluce.10", "evaluator": "transluce", "dimension": "G", "direction": "for", "claim": "Published independence policy aligned to AEF-1 2.3: annual disclosure of developer-linked revenue share, paid-evaluation disclosure, no organizational equity in developers, third-party ethics hotline.", "sources": [ "transluce-policy", "transluce-about" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "quote": "consistent with requirement 2.3 of the AEF-1 standard", "bound": { "floor": 3 }, "rule": "G.2", "quote_source": "transluce-policy", "graph_id": "signal#570", "source_graph_ids": [ "source#193", "source#189" ] }, { "id": "transluce.11", "evaluator": "transluce", "dimension": "P", "direction": "for", "claim": "Mandatory recusal of any employee with a significant financial interest in, or employment by, a system provider from evaluating that provider.", "sources": [ "transluce-policy" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "quote": "Transluce recuses any employee with a significant financial interest", "bound": { "floor": 3 }, "rule": "P.5", "quote_source": "transluce-policy", "graph_id": "signal#571", "source_graph_ids": [ "source#193" ] }, { "id": "transluce.12", "evaluator": "transluce", "dimension": "P", "direction": "against", "claim": "An advisor is on the technical staff of Thinking Machines; a board member runs Halcyon Futures, which funds AVERI and AIUC.", "sources": [ "transluce-about" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 2 }, "rule": "P.4", "quote": "Thinking Machines Board of Directors Alex Allain Co-Founder, U.S. Digital Response Mike McCormick CEO, Halcyon Futures", "quote_source": "transluce-about", "graph_id": "signal#572", "source_graph_ids": [ "source#189" ] } ], "scores": { "lab": 60, "regulator": 60, "public": 59, "equal": 63 }, "scores_by_policy": { "leads": { "lab": { "score": 60, "band": "clear", "coverage": 8, "range": [ 60, 60 ] }, "regulator": { "score": 60, "band": "clear", "coverage": 8, "range": [ 60, 60 ] }, "public": { "score": 59, "band": "clear", "coverage": 8, "range": [ 59, 59 ] }, "equal": { "score": 63, "band": "clear", "coverage": 8, "range": [ 63, 63 ] } }, "standard": { "lab": { "score": 60, "band": "clear", "coverage": 8, "range": [ 60, 60 ] }, "regulator": { "score": 60, "band": "clear", "coverage": 8, "range": [ 60, 60 ] }, "public": { "score": 59, "band": "clear", "coverage": 8, "range": [ 59, 59 ] }, "equal": { "score": 63, "band": "clear", "coverage": 8, "range": [ 63, 63 ] } }, "against_interest": { "lab": { "score": 52, "band": "clear", "coverage": 6, "range": [ 39, 64 ] }, "regulator": { "score": 50, "band": "clear", "coverage": 6, "range": [ 35, 65 ] }, "public": { "score": 51, "band": "clear", "coverage": 6, "range": [ 44, 59 ] }, "equal": { "score": 54, "band": "clear", "coverage": 6, "range": [ 41, 66 ] } }, "spans": { "lab": { "score": 60, "band": "clear", "coverage": 8, "range": [ 60, 60 ] }, "regulator": { "score": 60, "band": "clear", "coverage": 8, "range": [ 60, 60 ] }, "public": { "score": 59, "band": "clear", "coverage": 8, "range": [ 59, 59 ] }, "equal": { "score": 63, "band": "clear", "coverage": 8, "range": [ 63, 63 ] } }, "primary": { "lab": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "regulator": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "public": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] }, "equal": { "score": null, "band": "unevidenced", "coverage": 0, "range": [ 0, 100 ] } } }, "band": "clear", "coverage": 8, "floor": "F", "what_moves": { "lab": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 64, "band": "clear", "base": 60 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 62, "band": "clear", "base": 60 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 62, "band": "clear", "base": 60 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 65, "band": "clear", "base": 60 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 64, "band": "clear", "base": 60 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 63, "band": "clear", "base": 60 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 62, "band": "clear", "base": 60 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 61, "band": "clear", "base": 60 } ], "regulator": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 64, "band": "clear", "base": 60 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 63, "band": "clear", "base": 60 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 63, "band": "clear", "base": 60 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 65, "band": "clear", "base": 60 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 65, "band": "clear", "base": 60 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 64, "band": "clear", "base": 60 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 63, "band": "clear", "base": 60 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 60, "band": "clear", "base": 60 } ], "public": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 65, "band": "clear", "base": 59 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 63, "band": "clear", "base": 59 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 63, "band": "clear", "base": 59 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 60, "band": "clear", "base": 59 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 61, "band": "clear", "base": 59 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 64, "band": "clear", "base": 59 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 60, "band": "clear", "base": 59 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 60, "band": "clear", "base": 59 } ], "equal": [ { "dimension": "F", "from_value": 2, "to_value": 3, "score": 66, "band": "clear", "base": 63 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 66, "band": "clear", "base": 63 }, { "dimension": "P", "from_value": 2, "to_value": 3, "score": 66, "band": "clear", "base": 63 }, { "dimension": "A", "from_value": 2, "to_value": 3, "score": 66, "band": "clear", "base": 63 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 66, "band": "clear", "base": 63 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 66, "band": "clear", "base": 63 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 66, "band": "clear", "base": 63 }, { "dimension": "X", "from_value": 3, "to_value": 4, "score": 66, "band": "clear", "base": 63 } ] } }, { "id": "ukaisi", "name": "UK AI Security Institute", "type": "gov", "hq": "London, UK", "domains": [ "cyber", "bio", "jailbreak", "autonomy", "assurance" ], "confidence": "high", "summary": "Largest state evaluation team; 30+ models tested; Inspect open-sourced; Frontier AI Trends Report.", "what_would_move_the_score": "Statutory footing with power to compel testing and publish per-model findings.", "role": "government", "dissent": { "lower": "Publication (R) at 2 could be 1. The public record shows per-model findings from more than thirty tested systems have never been published; the only public output is one aggregated Trends Report, and CSA describes the underlying work as confidential between the Institute and individual labs. Under R.4 a public body that publishes nothing per model scores 1, and one aggregate report in two years is close to nothing per model.", "higher": "Publication (R) at 2 could be 3. The Trends Report published an adverse finding against every lab at once: a universal jailbreak in every system tested. The joint evaluations with CAISI produced public write-ups of ChatGPT Agent vulnerabilities and Constitutional Classifier bypasses. No lab-edited summary is documented, and the anchor-3 text (publishes, security-only redaction) is closer to that record than lab-edited summaries." }, "list_group": "government", "graph_id": "evaluator#229", "values": { "F": 3, "G": 3, "P": 3, "A": 3, "S": 3, "R": 2, "M": 3, "X": 4 }, "values_by_policy": { "leads": { "F": 3, "G": 3, "P": 3, "A": 3, "S": 3, "R": 2, "M": 3, "X": 4 }, "standard": { "F": 3, "G": 3, "P": 3, "A": 3, "S": 3, "R": 2, "M": 3, "X": 4 }, "against_interest": { "F": 3, "G": 3, "P": 3, "A": 3, "S": 3, "R": 2, "M": 3, "X": 4 }, "spans": { "F": 3, "G": 3, "P": 3, "A": 3, "S": 3, "R": 2, "M": 3, "X": 4 }, "primary": { "F": null, "G": 3, "P": 3, "A": null, "S": null, "R": null, "M": null, "X": null } }, "assessments": { "F": { "evaluator": "ukaisi", "dimension": "F", "value": 3, "anchor": 3, "signals": [ "ukaisi.01", "ukaisi.11", "ukaisi.12" ], "rationale": "Anchor 3: public core budget, but the institute runs a lab-funded grant pool, which is lab money passing through it even if not into its evaluation budget.", "assessed": "2026-09-15", "assessor": "yohei/claude v1.0", "open_questions": [ "Whether Alignment Project funds touch evaluation staff or infrastructure." ], "graph_id": "assessment#795", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "ukaisi.11", "ukaisi.12" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "ukaisi.11", "ukaisi.12" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "ukaisi.11", "ukaisi.12" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "ukaisi.11", "ukaisi.12" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "ukaisi.01", "ukaisi.11", "ukaisi.12" ] } }, "rationale_derived": "Capped at 3 by ukaisi.11 (F.9): AISI administers the Alignment Project, a GBP 27m grant pool that includes GBP 5.6m from OpenAI and money from Anthropic, Microsoft and AWS; core budget of about GBP 66m per year is public.; capped at 3 by ukaisi.12 (F.9): The Alignment Project took £5.6m from OpenAI. The £27m/£12m totals, the wider backer list, and OpenAI’s non-influence statement are per the cited sources; spans not yet captured.. Floors up to 3 from ukaisi.01 do not exceed the cap.", "rationale_leads": "Capped at 3 by ukaisi.11 (F.9): AISI administers the Alignment Project, a GBP 27m grant pool that includes GBP 5.6m from OpenAI and money from Anthropic, Microsoft and AWS; core budget of about GBP 66m per year is public.; capped at 3 by ukaisi.12 (F.9): The Alignment Project took £5.6m from OpenAI. The £27m/£12m totals, the wider backer list, and OpenAI’s non-influence statement are per the cited sources; spans not yet captured.. Floors up to 3 from ukaisi.01 do not exceed the cap." }, "G": { "evaluator": "ukaisi", "dimension": "G", "value": 3, "anchor": 3, "signals": [ "ukaisi.08" ], "rationale": "Anchor 3: public body with civil-service conflict rules (ukaisi.08). No tier-1 source for an independent board or external review, so 4 is unearned under the tier rule (C10).", "assessed": "2026-09-15", "assessor": "yohei/claude v0", "graph_id": "assessment#796", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 3, "b": [ "ukaisi.08" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "ukaisi.08" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "ukaisi.08" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "ukaisi.08" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": 3, "b": [ "ukaisi.08" ], "h": false, "c": false, "u": false, "x": [] } }, "rationale_derived": "Floored at 3 by ukaisi.08 (G.2): Public body with civil-service conflict rules.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by ukaisi.08 (G.2): Public body with civil-service conflict rules.. No admissible signal caps it." }, "P": { "evaluator": "ukaisi", "dimension": "P", "value": 3, "anchor": 3, "signals": [ "ukaisi.07", "ukaisi.13" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#797", "evidence_tier": "tier 1 (filing/index)", "derived": { "leads": { "v": 3, "b": [ "ukaisi.13" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "ukaisi.13" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "ukaisi.13" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "ukaisi.13" ], "h": false, "c": false, "u": false, "x": [ "ukaisi.07" ] }, "primary": { "v": 3, "b": [ "ukaisi.13" ], "h": false, "c": false, "u": false, "x": [ "ukaisi.07" ] } }, "rationale_derived": "Floored at 3 by ukaisi.13 (P.5): AISI staff are civil servants in DSIT and are bound by the Civil Service Code (conflicts and gifts) and the Business Appointment Rules, a statutory post-employment rule that applies for one to two years after leaving Crown service.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by ukaisi.13 (P.5): AISI staff are civil servants in DSIT and are bound by the Civil Service Code (conflicts and gifts) and the Business Appointment Rules, a statutory post-employment rule that applies for one to two years after leaving Crown service.. No admissible signal caps it." }, "A": { "evaluator": "ukaisi", "dimension": "A", "value": 3, "anchor": 3, "signals": [ "ukaisi.05", "ukaisi.06" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "mechanism": "lab-controlled", "graph_id": "assessment#798", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 3, "b": [ "ukaisi.06" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "ukaisi.06" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "ukaisi.06" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "ukaisi.06" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "ukaisi.05", "ukaisi.06" ] } }, "rationale_derived": "Floored at 3 by ukaisi.06 (A.4): Joint testing with the US institute; agreements with OpenAI, Anthropic, Google, Microsoft, Cohere; 30+ models tested.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by ukaisi.06 (A.4): Joint testing with the US institute; agreements with OpenAI, Anthropic, Google, Microsoft, Cohere; 30+ models tested.. No admissible signal caps it." }, "S": { "evaluator": "ukaisi", "dimension": "S", "value": 3, "anchor": 3, "signals": [ "ukaisi.09", "ukaisi.14" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#799", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 3, "b": [ "ukaisi.14" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "ukaisi.14" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "ukaisi.14" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "ukaisi.14" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "ukaisi.09", "ukaisi.14" ] } }, "rationale_derived": "Capped at 3 by ukaisi.14 (S.4): No statutory power to define scope or compel testing; the institute operates through voluntary testing agreements backed by bilateral memoranda of understanding, and the Frontier AI Bill had not been introduced as of September 2026.. Floors up to 3 from ukaisi.09 do not exceed the cap.", "rationale_leads": "Capped at 3 by ukaisi.14 (S.4): No statutory power to define scope or compel testing; the institute operates through voluntary testing agreements backed by bilateral memoranda of understanding, and the Frontier AI Bill had not been introduced as of September 2026.. Floors up to 3 from ukaisi.09 do not exceed the cap." }, "R": { "evaluator": "ukaisi", "dimension": "R", "value": 2, "anchor": 2, "signals": [ "ukaisi.03", "ukaisi.04" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "mechanism": "statutory", "graph_id": "assessment#800", "evidence_tier": "tier 4 (press)", "derived": { "leads": { "v": 2, "b": [ "ukaisi.04" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 2, "b": [ "ukaisi.04" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 2, "b": [ "ukaisi.04" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 2, "b": [ "ukaisi.04" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "ukaisi.03", "ukaisi.04" ] } }, "rationale_derived": "Capped at 2 by ukaisi.04 (R.4): Most per-model results stay confidential; the public sees aggregated trends.. Floors up to 2 from ukaisi.03 do not exceed the cap.", "rationale_leads": "Capped at 2 by ukaisi.04 (R.4): Most per-model results stay confidential; the public sees aggregated trends.. Floors up to 2 from ukaisi.03 do not exceed the cap." }, "M": { "evaluator": "ukaisi", "dimension": "M", "value": 3, "anchor": 3, "signals": [ "ukaisi.02" ], "assessed": "2026-09-15", "assessor": "rules v0.1 (RULES.md)", "graph_id": "assessment#801", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 3, "b": [ "ukaisi.02" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 3, "b": [ "ukaisi.02" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 3, "b": [ "ukaisi.02" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 3, "b": [ "ukaisi.02" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "ukaisi.02" ] } }, "rationale_derived": "Floored at 3 by ukaisi.02 (M.1): Open-source Inspect framework and published methodology.. No admissible signal caps it.", "rationale_leads": "Floored at 3 by ukaisi.02 (M.1): Open-source Inspect framework and published methodology.. No admissible signal caps it." }, "X": { "evaluator": "ukaisi", "dimension": "X", "value": 4, "anchor": 4, "signals": [ "ukaisi.10" ], "assessed": "2026-09-14", "assessor": "yohei/claude v0", "graph_id": "assessment#802", "evidence_tier": "tier 3 (self)", "derived": { "leads": { "v": 4, "b": [ "ukaisi.10" ], "h": false, "c": false, "u": false, "x": [] }, "standard": { "v": 4, "b": [ "ukaisi.10" ], "h": false, "c": false, "u": false, "x": [] }, "against_interest": { "v": 4, "b": [ "ukaisi.10" ], "h": false, "c": false, "u": false, "x": [] }, "spans": { "v": 4, "b": [ "ukaisi.10" ], "h": false, "c": false, "u": false, "x": [] }, "primary": { "v": null, "b": [], "h": false, "c": false, "u": true, "x": [ "ukaisi.10" ] } }, "rationale_derived": "Floored at 4 by ukaisi.10 (X.7): No commercial products.. No admissible signal caps it.", "rationale_leads": "Floored at 4 by ukaisi.10 (X.7): No commercial products.. No admissible signal caps it." } }, "signals": [ { "id": "ukaisi.01", "evaluator": "ukaisi", "dimension": "F", "direction": "for", "claim": "Taxpayer funded; no commercial relationship with labs.", "sources": [ "regulations-ai-aisi", "ukaisi-about" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "F.9", "quote": "internal technical research directorate within the Department for Science, Innovation and Technology", "quote_source": "regulations-ai-aisi", "graph_id": "signal#573", "source_graph_ids": [ "source#161", "source#194" ] }, { "id": "ukaisi.02", "evaluator": "ukaisi", "dimension": "M", "direction": "for", "claim": "Open-source Inspect framework and published methodology.", "sources": [ "csa-aisi-trends", "ukaisi-inspect" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "M.1", "quote": "An open-source framework for large language model evaluations", "quote_source": "ukaisi-inspect", "graph_id": "signal#574", "source_graph_ids": [ "source#55", "source#197" ] }, { "id": "ukaisi.03", "evaluator": "ukaisi", "dimension": "R", "direction": "for", "claim": "Found universal jailbreaks in every system tested and published that in the Frontier AI Trends Report.", "sources": [ "csa-aisi-trends" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "quote": "AISI found universal jailbreaks across every tested system", "bound": { "floor": 2 }, "rule": "R.4", "quote_source": "csa-aisi-trends", "graph_id": "signal#575", "source_graph_ids": [ "source#55" ] }, { "id": "ukaisi.04", "evaluator": "ukaisi", "dimension": "R", "direction": "against", "claim": "Most per-model results stay confidential; the public sees aggregated trends.", "sources": [ "csa-aisi-trends" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "cap": 2 }, "rule": "R.4", "quote": "evaluation work that would otherwise remain confidential between the Institute and individual AI labs", "quote_source": "csa-aisi-trends", "graph_id": "signal#576", "source_graph_ids": [ "source#55" ] }, { "id": "ukaisi.05", "evaluator": "ukaisi", "dimension": "A", "direction": "against", "claim": "No statutory powers; access rests on MOUs any lab could decline to renew; Frontier AI Bill not introduced as of September 2026.", "sources": [ "csa-aisi-trends", "regulations-ai-aisi" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": null, "rule": "A.3", "bound_note": "A.3: \"the lab can decline to renew\" is recorded as a mechanism (lab-controlled), not as a cap.", "quote": "the bill had not yet been introduced to Parliament", "quote_source": "csa-aisi-trends", "graph_id": "signal#577", "source_graph_ids": [ "source#55", "source#161" ] }, { "id": "ukaisi.06", "evaluator": "ukaisi", "dimension": "A", "direction": "for", "claim": "Joint testing with the US institute; agreements with OpenAI, Anthropic, Google, Microsoft, Cohere; 30+ models tested.", "sources": [ "aiwiki-ukaisi", "csa-aisi-trends" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "A.4", "quote": "running pre-deployment and post-deployment evaluations of the most capable models released by major AI developers", "quote_source": "csa-aisi-trends", "graph_id": "signal#578", "source_graph_ids": [ "source#14", "source#55" ] }, { "id": "ukaisi.07", "evaluator": "ukaisi", "dimension": "P", "direction": "against", "claim": "Staff move between the institute and labs in both directions.", "sources": [ "csa-aisi-trends" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": null, "rule": "P.3", "bound_note": "P.3: two-way hiring caps at 2, but \"a statutory post-employment rule for the body's staff lifts the cap\"; the Business Appointment Rules (new signal ukaisi.13) lift it. Absent ukaisi.13 this signal caps P at 2.", "graph_id": "signal#579", "source_graph_ids": [ "source#55" ] }, { "id": "ukaisi.08", "evaluator": "ukaisi", "dimension": "G", "direction": "for", "claim": "Public body with civil-service conflict rules.", "sources": [ "regulations-ai-aisi", "ukaisi-civil-service-code" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "G.2", "quote": "The Institute is an advisory technical research body rather than a formal regulator", "quote_source": "regulations-ai-aisi", "graph_id": "signal#580", "source_graph_ids": [ "source#161", "source#196" ] }, { "id": "ukaisi.09", "evaluator": "ukaisi", "dimension": "S", "direction": "for", "claim": "Sets its own evaluation agenda within MOU terms.", "sources": [ "csa-aisi-trends" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 3 }, "rule": "S.4", "quote": "AISI’s voluntary testing arrangements with frontier developers remain the primary mechanism", "quote_source": "csa-aisi-trends", "graph_id": "signal#581", "source_graph_ids": [ "source#55" ] }, { "id": "ukaisi.10", "evaluator": "ukaisi", "dimension": "X", "direction": "for", "claim": "No commercial products.", "sources": [ "regulations-ai-aisi", "ukaisi-about" ], "recorded": "2026-09-14", "curator": "yohei/claude v0", "bound": { "floor": 4 }, "rule": "X.7", "quote": "The Institute is an advisory technical research body rather than a formal regulator", "quote_source": "regulations-ai-aisi", "graph_id": "signal#582", "source_graph_ids": [ "source#161", "source#194" ] }, { "id": "ukaisi.11", "evaluator": "ukaisi", "dimension": "F", "direction": "against", "claim": "AISI administers the Alignment Project, a GBP 27m grant pool that includes GBP 5.6m from OpenAI and money from Anthropic, Microsoft and AWS; core budget of about GBP 66m per year is public.", "sources": [ "aisi-grants", "alignment-project-about", "ukaisi-about" ], "recorded": "2026-09-14", "curator": "yohei/claude v0.4", "as_of": "2026-09-14", "bound": { "cap": 3 }, "rule": "F.9", "quote": "OpenAI, Microsoft, Amazon Web Services (AWS), Schmidt Sciences, Anthropic", "quote_source": "alignment-project-about", "graph_id": "signal#583", "source_graph_ids": [ "source#13", "source#15", "source#194" ] }, { "id": "ukaisi.12", "evaluator": "ukaisi", "dimension": "F", "direction": "against", "claim": "The Alignment Project took £5.6m from OpenAI. The £27m/£12m totals, the wider backer list, and OpenAI’s non-influence statement are per the cited sources; spans not yet captured.", "sources": [ "aisi-alignment-60", "alignment-project-about", "openai-alignment-project" ], "recorded": "2026-09-15", "curator": "yohei/claude v1.0", "as_of": "2026-02", "quote": "including £5.6m from OpenAI", "bound": { "cap": 3 }, "rule": "F.9", "quote_source": "aisi-alignment-60", "graph_id": "signal#584", "source_graph_ids": [ "source#12", "source#15", "source#141" ] }, { "id": "ukaisi.13", "evaluator": "ukaisi", "dimension": "P", "direction": "for", "claim": "AISI staff are civil servants in DSIT and are bound by the Civil Service Code (conflicts and gifts) and the Business Appointment Rules, a statutory post-employment rule that applies for one to two years after leaving Crown service.", "sources": [ "ukaisi-civil-service-code", "ukaisi-bar-crown", "ukaisi-about" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "floor": 3 }, "rule": "P.5", "quote": "two years after leaving Crown service, if you are grade SCS1 or SCS2", "quote_source": "ukaisi-bar-crown", "note": "Statutory post-employment rules count as cooling-off (P.3) and lift the two-way-hiring cap; the Code is a published mandatory conflict rule (P.5). Individual lab ties are declared to the department, not published, so 3 not 4 (P.8 needs disclosed ties and no equity with tier-1 evidence).", "graph_id": "signal#585", "source_graph_ids": [ "source#196", "source#195", "source#194" ] }, { "id": "ukaisi.14", "evaluator": "ukaisi", "dimension": "S", "direction": "against", "claim": "No statutory power to define scope or compel testing; the institute operates through voluntary testing agreements backed by bilateral memoranda of understanding, and the Frontier AI Bill had not been introduced as of September 2026.", "sources": [ "regulations-ai-aisi", "csa-aisi-trends" ], "recorded": "2026-09-15", "curator": "yohei/claude v0.5", "bound": { "cap": 3 }, "rule": "S.4", "quote": "The institute cannot fine, audit, or directly penalise model developers", "quote_source": "regulations-ai-aisi", "note": "Voluntary memoranda: 3 (S.4). Statutory scope power would be 4.", "graph_id": "signal#586", "source_graph_ids": [ "source#161", "source#55" ] } ], "scores": { "lab": 73, "regulator": 71, "public": 71, "equal": 75 }, "scores_by_policy": { "leads": { "lab": { "score": 73, "band": "clear", "coverage": 8, "range": [ 73, 73 ] }, "regulator": { "score": 71, "band": "clear", "coverage": 8, "range": [ 71, 71 ] }, "public": { "score": 71, "band": "clear", "coverage": 8, "range": [ 71, 71 ] }, "equal": { "score": 75, "band": "clear", "coverage": 8, "range": [ 75, 75 ] } }, "standard": { "lab": { "score": 73, "band": "clear", "coverage": 8, "range": [ 73, 73 ] }, "regulator": { "score": 71, "band": "clear", "coverage": 8, "range": [ 71, 71 ] }, "public": { "score": 71, "band": "clear", "coverage": 8, "range": [ 71, 71 ] }, "equal": { "score": 75, "band": "clear", "coverage": 8, "range": [ 75, 75 ] } }, "against_interest": { "lab": { "score": 73, "band": "clear", "coverage": 8, "range": [ 73, 73 ] }, "regulator": { "score": 71, "band": "clear", "coverage": 8, "range": [ 71, 71 ] }, "public": { "score": 71, "band": "clear", "coverage": 8, "range": [ 71, 71 ] }, "equal": { "score": 75, "band": "clear", "coverage": 8, "range": [ 75, 75 ] } }, "spans": { "lab": { "score": 73, "band": "clear", "coverage": 8, "range": [ 73, 73 ] }, "regulator": { "score": 71, "band": "clear", "coverage": 8, "range": [ 71, 71 ] }, "public": { "score": 71, "band": "clear", "coverage": 8, "range": [ 71, 71 ] }, "equal": { "score": 75, "band": "clear", "coverage": 8, "range": [ 75, 75 ] } }, "primary": { "lab": { "score": 75, "band": "clear", "coverage": 2, "range": [ 14, 96 ] }, "regulator": { "score": 75, "band": "clear", "coverage": 2, "range": [ 15, 95 ] }, "public": { "score": 75, "band": "clear", "coverage": 2, "range": [ 23, 93 ] }, "equal": { "score": 75, "band": "clear", "coverage": 2, "range": [ 19, 94 ] } } }, "band": "clear", "coverage": 8, "floor": "R", "what_moves": { "lab": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 77, "band": "clear", "base": 73 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 73 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 73 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 73 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 77, "band": "clear", "base": 73 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 76, "band": "clear", "base": 73 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 73 } ], "regulator": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 71 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 74, "band": "clear", "base": 71 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 74, "band": "clear", "base": 71 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 76, "band": "clear", "base": 71 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 76, "band": "clear", "base": 71 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 75, "band": "clear", "base": 71 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 74, "band": "clear", "base": 71 } ], "public": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 71 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 71 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 75, "band": "clear", "base": 71 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 73, "band": "clear", "base": 71 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 74, "band": "clear", "base": 71 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 76, "band": "clear", "base": 71 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 73, "band": "clear", "base": 71 } ], "equal": [ { "dimension": "F", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 75 }, { "dimension": "G", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 75 }, { "dimension": "P", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 75 }, { "dimension": "A", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 75 }, { "dimension": "S", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 75 }, { "dimension": "R", "from_value": 2, "to_value": 3, "score": 78, "band": "clear", "base": 75 }, { "dimension": "M", "from_value": 3, "to_value": 4, "score": 78, "band": "clear", "base": 75 } ] } } ], "ranked_ids": [ "andon", "apollo", "averi", "cais", "caisi", "dreadnode", "epoch", "equistamp", "euaio", "farai", "grayswan", "hal", "humane", "irregular", "metr", "mlcommons", "msft", "nemesys", "palisade", "rand", "redwood", "saferai", "scale", "securebio", "transluce", "ukaisi" ] }